From c6e91ff15a71c9ce2f09da73fb6207650b9367e4 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Feb 2021 12:27:27 +1000 Subject: [PATCH 1/6] sync mock server with the one from ckanext-archiver, including fixing text vs binary typing --- ckanext/qa/tests/mock_remote_server.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ckanext/qa/tests/mock_remote_server.py b/ckanext/qa/tests/mock_remote_server.py index 9c59ed5f..3761a90d 100644 --- a/ckanext/qa/tests/mock_remote_server.py +++ b/ckanext/qa/tests/mock_remote_server.py @@ -97,7 +97,8 @@ class MockEchoTestServer(MockHTTPServer): a 500 error response: 'http://localhost/?status=500' a 200 OK response, returning the function's docstring: - 'http://localhost/?status=200;content-type=text/plain;content_var=ckan.tests.lib.test_package_search:test_wsgi_app.__doc__' + 'http://localhost/?status=200;content-type=text/plain;content_var + =ckan.tests.lib.test_package_search:test_wsgi_app.__doc__' To specify content, use: @@ -114,10 +115,16 @@ def __call__(self, environ, start_response): if 'content_var' in request.str_params: content = request.str_params.get('content_var') content = self.get_content(content) + elif 'content_long' in request.str_params: + content = '*' * 1000001 else: content = request.str_params.get('content', '') + if 'method' in request.str_params \ + and request.method.lower() != request.str_params['method'].lower(): + content = '' + status = 405 - if isinstance(content, six.string_types): + if isinstance(content, six.text_type): raise TypeError("Expected raw byte string for content") headers = [ @@ -125,8 +132,11 @@ def __call__(self, environ, start_response): for item in request.str_params.items() if item[0] not in ('content', 'status') ] - if content: - headers += [('Content-Length', str(len(content)))] + if 'length' in request.str_params: + cl = request.str_params.get('length') + headers += [('Content-Length', cl)] + elif content and 'no-content-length' not in request.str_params: + headers += [('Content-Length', six.binary_type(len(content)))] start_response( '%d %s' % (status, responses[status]), headers From 0b49e96a8cf4c2ff2dbadcf81fa8b359064cf0b4 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Feb 2021 12:43:11 +1000 Subject: [PATCH 2/6] move plugin list into test.ini --- test-core.ini | 4 +--- test.ini | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-core.ini b/test-core.ini index 5996c2d8..04f2d7a8 100644 --- a/test-core.ini +++ b/test-core.ini @@ -21,8 +21,6 @@ port = 5000 [app:main] use = config:../ckan/test-core.ini -ckan.plugins = qa - # Logging configuration [loggers] keys = root, ckan, ckanext, sqlalchemy @@ -52,7 +50,7 @@ propagate = 0 [logger_sqlalchemy] handlers = qualname = sqlalchemy.engine -level = WARN +level = WARN propagate = 0 [handler_console] diff --git a/test.ini b/test.ini index 7c1094c4..eee17f3d 100644 --- a/test.ini +++ b/test.ini @@ -22,6 +22,8 @@ use = config:test-core.ini # run fast. faster_db_test_hacks = True sqlalchemy.url = sqlite:/// + +ckan.plugins = qa # NB: other test configuration should go in test-core.ini, which is # what the postgres tests use. @@ -54,7 +56,7 @@ propagate = 0 [logger_sqlalchemy] handlers = console qualname = sqlalchemy.engine -level = WARN +level = WARN [logger_harvest] level = WARNING From c06a2c9aae003705d554f351bea8ba28df6a629d Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Feb 2021 13:02:35 +1000 Subject: [PATCH 3/6] improve assertion error message --- ckanext/qa/tests/test_sniff_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/qa/tests/test_sniff_format.py b/ckanext/qa/tests/test_sniff_format.py index c3b7f279..beb7b7b5 100644 --- a/ckanext/qa/tests/test_sniff_format.py +++ b/ckanext/qa/tests/test_sniff_format.py @@ -30,7 +30,7 @@ def assert_file_has_format_sniffed_correctly(cls, format_extension, filepath): '''Given a filepath, checks the sniffed format matches the format_extension.''' expected_format = format_extension sniffed_format = sniff_file_format(filepath) - assert sniffed_format, expected_format + assert sniffed_format, "Expected {} but failed to sniff any format: {}".format(expected_format, sniffed_format) expected_format_without_zip = expected_format.replace('.zip', '') assert_equal(sniffed_format['format'].lower(), expected_format_without_zip) From 7435f33c2e3395f2e62c76418508a4b3276da102 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Feb 2021 13:27:45 +1000 Subject: [PATCH 4/6] improve error messages for testing file sniffing --- ckanext/qa/tests/test_sniff_format.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ckanext/qa/tests/test_sniff_format.py b/ckanext/qa/tests/test_sniff_format.py index beb7b7b5..2d1a35de 100644 --- a/ckanext/qa/tests/test_sniff_format.py +++ b/ckanext/qa/tests/test_sniff_format.py @@ -30,7 +30,7 @@ def assert_file_has_format_sniffed_correctly(cls, format_extension, filepath): '''Given a filepath, checks the sniffed format matches the format_extension.''' expected_format = format_extension sniffed_format = sniff_file_format(filepath) - assert sniffed_format, "Expected {} but failed to sniff any format: {}".format(expected_format, sniffed_format) + assert sniffed_format, "Expected {} but failed to sniff any format for file: {}".format(expected_format, filepath) expected_format_without_zip = expected_format.replace('.zip', '') assert_equal(sniffed_format['format'].lower(), expected_format_without_zip) @@ -49,16 +49,11 @@ def assert_file_has_format_sniffed_correctly(cls, format_extension, filepath): def check_format(cls, format, filename=None): for format_extension, filepath in cls.fixture_files: if format_extension == format: - if filename: - if filename in filepath: - break - else: - continue - else: - break + if not filename or filename in filepath: + cls.assert_file_has_format_sniffed_correctly(format_extension, filepath) + break else: assert 0, format # Could not find fixture for format - cls.assert_file_has_format_sniffed_correctly(format_extension, filepath) def test_xls(self): self.check_format('xls', '10-p108-data-results') From b656376116c07a30e884909d0b94f6d9a94e2857 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Feb 2021 13:40:53 +1000 Subject: [PATCH 5/6] oops fix loop termination logic --- ckanext/qa/tests/test_sniff_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/qa/tests/test_sniff_format.py b/ckanext/qa/tests/test_sniff_format.py index 2d1a35de..86039bc8 100644 --- a/ckanext/qa/tests/test_sniff_format.py +++ b/ckanext/qa/tests/test_sniff_format.py @@ -51,7 +51,7 @@ def check_format(cls, format, filename=None): if format_extension == format: if not filename or filename in filepath: cls.assert_file_has_format_sniffed_correctly(format_extension, filepath) - break + break else: assert 0, format # Could not find fixture for format From 85f426111528cadfad9aa64565da90f7e918e988 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Feb 2021 14:53:46 +1000 Subject: [PATCH 6/6] use BSD 'file' fallback more consistently when other methods fail --- ckanext/qa/sniff_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/qa/sniff_format.py b/ckanext/qa/sniff_format.py index e1d5869a..2a3b8a20 100644 --- a/ckanext/qa/sniff_format.py +++ b/ckanext/qa/sniff_format.py @@ -124,7 +124,7 @@ def sniff_file_format(filepath): if has_rdfa(buf): format_ = {'format': 'RDFa'} - else: + if not format_: # Excel files sometimes not picked up by magic, so try alternative if is_excel(filepath): format_ = {'format': 'XLS'}