From ebae7577f22c2fd427cd88e931f2c7a7e69031d9 Mon Sep 17 00:00:00 2001 From: SharonGoliath Date: Tue, 1 Jun 2021 10:47:57 -0700 Subject: [PATCH] =?UTF-8?q?CADC-9668=20-=20archive.gemini.edu/fullheaders/?= =?UTF-8?q?=20will=20sometimes=20=E2=80=A6=20(#146)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CADC-9668 - archive.gemini.edu/fullheaders/ will sometimes return a 403. --- caom2utils/caom2utils/fits2caom2.py | 38 ++++++++---- .../caom2utils/tests/test_fits2caom2.py | 58 +++++++++++++++++++ 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/caom2utils/caom2utils/fits2caom2.py b/caom2utils/caom2utils/fits2caom2.py index 630d1030..c7a86a2d 100755 --- a/caom2utils/caom2utils/fits2caom2.py +++ b/caom2utils/caom2utils/fits2caom2.py @@ -1732,6 +1732,9 @@ def _execute_external(self, value, key, extension): elif isinstance(self, FitsParser): parameter = {'uri': self.uri, 'header': self._headers[extension]} + else: + parameter = {'uri': self.uri, + 'header': None} result = '' execute = None @@ -2851,11 +2854,14 @@ def _get_from_list(self, lookup, index, current=None): if keywords[0].index(ii) == len(keywords[0]) - 1: self.add_error(lookup, sys.exc_info()[1]) # assign a default value, if one exists - if keywords[1] and current is None: - value = keywords[1] - self.logger.debug( - '{}: assigned default value {}.'.format(lookup, - value)) + if keywords[1]: + if current is None: + value = keywords[1] + self.logger.debug( + '{}: assigned default value {}.'.format(lookup, + value)) + else: + value = current if value is None: # checking current does not work in the general case, # because current might legitimately be 'None' @@ -2867,11 +2873,14 @@ def _get_from_list(self, lookup, index, current=None): '{!r}.'.format(lookup, value)) else: # assign a default value, if one exists - if keywords[1] and current is None: - value = keywords[1] - self.logger.debug( - '{}: assigned default value {}.'.format(lookup, - value)) + if keywords[1]: + if current is None: + value = keywords[1] + self.logger.debug( + '{}: assigned default value {}.'.format(lookup, + value)) + else: + value = current elif (keywords is not None) and (keywords != ''): value = keywords @@ -3628,6 +3637,7 @@ def get_cadc_headers(uri, subject=None): of astropy.wcs.Header type - essentially a dictionary of FITS keywords. """ file_url = urlparse(uri) + # TODO if file_url.scheme == 'cadc', need to get vos headers if (file_url.scheme == 'ad' or file_url.scheme == 'gemini'): # create possible types of subjects @@ -3762,6 +3772,8 @@ def _update_artifact_meta(uri, artifact, subject=None, connected=True): :return: """ file_url = urlparse(uri) + # TODO - if file_url.scheme == 'cadc', need to use vos to get + # artifact metadata if file_url.scheme == 'ad': metadata = _get_cadc_meta(subject, file_url.path) elif file_url.scheme == 'gemini': @@ -3982,7 +3994,11 @@ def _augment(obs, product_id, uri, blueprint, subject, dumpconfig=False, elif external_url: headers = get_external_headers(external_url) if headers is None: - parser = None + logging.debug( + 'Using a GenericParser for un-retrievable remote headers ' + '{}'.format(uri) + ) + parser = GenericParser(blueprint, uri=uri) else: logging.debug( 'Using a FitsParser for remote headers {}'.format(uri)) diff --git a/caom2utils/caom2utils/tests/test_fits2caom2.py b/caom2utils/caom2utils/tests/test_fits2caom2.py index a3f5da92..18c015b9 100755 --- a/caom2utils/caom2utils/tests/test_fits2caom2.py +++ b/caom2utils/caom2utils/tests/test_fits2caom2.py @@ -94,6 +94,7 @@ from mock import Mock, patch from six import StringIO, BytesIO +import importlib import os import sys @@ -1262,6 +1263,32 @@ def test_get_external_headers(): assert session_get_mock.is_called_with(test_uri) +@patch('caom2utils.fits2caom2.get_external_headers') +def test_get_external_headers_fails(get_external_mock): + get_external_mock.return_value = None + test_collection = 'TEST_COLLECTION' + test_obs_id = 'TEST_OBS_ID' + test_uri = 'gemini:{}/abc.fits'.format(test_collection) + test_product_id = 'TEST_PRODUCT_ID' + test_blueprint = caom2utils.fits2caom2.ObsBlueprint() + test_observation = SimpleObservation(collection=test_collection, + observation_id=test_obs_id, + algorithm=Algorithm(name='exposure')) + test_result = caom2utils.fits2caom2._augment( + obs=test_observation, + product_id=test_product_id, + uri=test_uri, + blueprint=test_blueprint, + subject=net.Subject(), + external_url='https://localhost/files/test_file.fits.gz', + ) + assert test_result is not None, 'expect a result' + assert len(test_result.planes.values()) == 1, 'plane added to result' + test_plane = test_result.planes[test_product_id] + assert len(test_plane.artifacts.values()) == 1, 'artifact added to plane' + assert test_uri in test_plane.artifacts.keys(), 'wrong artifact uri' + + def test_apply_blueprint(): # test a Gemini case where there are two keywords, one each for # different instruments, and the default ends up getting set when the @@ -1300,6 +1327,29 @@ def test_apply_blueprint(): test_parser._headers[0]['IMAGESWV'], 'should not be set' +def test_apply_blueprint_execute_external(): + test_module = importlib.import_module(__name__) + test_generic_blueprint = ObsBlueprint(module=test_module) + test_generic_blueprint.set('Observation.type', '_get_test_obs_type(parameters)') + + # generic parser + test_generic_parser = GenericParser(test_generic_blueprint) + assert test_generic_parser is not None, \ + 'expect generic construction to complete' + assert test_generic_parser._get_from_list('Observation.type', index=0) \ + == 'generic_parser_value', 'wrong generic plan value' + + # fits parser + test_fits_blueprint = ObsBlueprint(module=test_module) + test_fits_blueprint.set('Observation.type', '_get_test_obs_type(parameters)') + test_fits_parser = FitsParser(src=sample_file_4axes, + obs_blueprint=test_fits_blueprint) + assert test_fits_parser is not None, \ + 'expect fits construction to complete' + assert test_fits_parser._get_from_list('Observation.type', index=0) \ + == 'fits_parser_value', 'wrong fits plan value' + + def test_update_artifact_meta_errors(): test_uri = 'gemini:GEMINI/abc.jpg' test_artifact = Artifact(uri=test_uri, @@ -1415,3 +1465,11 @@ def _get_node(uri, limit, force): node.props = {'MD5': '5b00b00d4b06aba986c3663d09aa581f', 'length': 682560} return node + + +def _get_test_obs_type(parameters): + assert parameters is not None + if parameters.get('header') is None: + return 'generic_parser_value' + else: + return 'fits_parser_value'