CADC-9668 - archive.gemini.edu/fullheaders/<filename> will sometimes … (

#146)

* CADC-9668 - archive.gemini.edu/fullheaders/<filename> will sometimes return a 403.

caom2utils/caom2utils/fits2caom2.py

@@ -1732,6 +1732,9 @@ def _execute_external(self, value, key, extension):
         elif isinstance(self, FitsParser):
             parameter = {'uri': self.uri,
                          'header': self._headers[extension]}
+        else:
+            parameter = {'uri': self.uri,
+                         'header': None}
 
         result = ''
         execute = None
@@ -2851,11 +2854,14 @@ def _get_from_list(self, lookup, index, current=None):
                     if keywords[0].index(ii) == len(keywords[0]) - 1:
                         self.add_error(lookup, sys.exc_info()[1])
                     # assign a default value, if one exists
-                    if keywords[1] and current is None:
-                        value = keywords[1]
-                        self.logger.debug(
-                            '{}: assigned default value {}.'.format(lookup,
-                                                                    value))
+                    if keywords[1]:
+                        if current is None:
+                            value = keywords[1]
+                            self.logger.debug(
+                                '{}: assigned default value {}.'.format(lookup,
+                                                                        value))
+                        else:
+                            value = current
             if value is None:
                 # checking current does not work in the general case,
                 # because current might legitimately be 'None'
@@ -2867,11 +2873,14 @@ def _get_from_list(self, lookup, index, current=None):
                                           '{!r}.'.format(lookup, value))
                 else:
                     # assign a default value, if one exists
-                    if keywords[1] and current is None:
-                        value = keywords[1]
-                        self.logger.debug(
-                            '{}: assigned default value {}.'.format(lookup,
-                                                                    value))
+                    if keywords[1]:
+                        if current is None:
+                            value = keywords[1]
+                            self.logger.debug(
+                                '{}: assigned default value {}.'.format(lookup,
+                                                                        value))
+                        else:
+                            value = current
 
         elif (keywords is not None) and (keywords != ''):
             value = keywords
@@ -3628,6 +3637,7 @@ def get_cadc_headers(uri, subject=None):
     of astropy.wcs.Header type - essentially a dictionary of FITS keywords.
     """
     file_url = urlparse(uri)
+    # TODO if file_url.scheme == 'cadc', need to get vos headers
     if (file_url.scheme == 'ad' or
             file_url.scheme == 'gemini'):
         # create possible types of subjects
@@ -3762,6 +3772,8 @@ def _update_artifact_meta(uri, artifact, subject=None, connected=True):
     :return:
     """
     file_url = urlparse(uri)
+    # TODO - if file_url.scheme == 'cadc', need to use vos to get
+    # artifact metadata
     if file_url.scheme == 'ad':
         metadata = _get_cadc_meta(subject, file_url.path)
     elif file_url.scheme == 'gemini':
@@ -3982,7 +3994,11 @@ def _augment(obs, product_id, uri, blueprint, subject, dumpconfig=False,
     elif external_url:
         headers = get_external_headers(external_url)
         if headers is None:
-            parser = None
+            logging.debug(
+                'Using a GenericParser for un-retrievable remote headers '
+                '{}'.format(uri)
+            )
+            parser = GenericParser(blueprint, uri=uri)
         else:
             logging.debug(
                 'Using a FitsParser for remote headers {}'.format(uri))

caom2utils/caom2utils/tests/test_fits2caom2.py

@@ -94,6 +94,7 @@
 from mock import Mock, patch
 from six import StringIO, BytesIO
 
+import importlib
 import os
 import sys
 
@@ -1262,6 +1263,32 @@ def test_get_external_headers():
         assert session_get_mock.is_called_with(test_uri)
 
 
+@patch('caom2utils.fits2caom2.get_external_headers')
+def test_get_external_headers_fails(get_external_mock):
+    get_external_mock.return_value = None
+    test_collection = 'TEST_COLLECTION'
+    test_obs_id = 'TEST_OBS_ID'
+    test_uri = 'gemini:{}/abc.fits'.format(test_collection)
+    test_product_id = 'TEST_PRODUCT_ID'
+    test_blueprint = caom2utils.fits2caom2.ObsBlueprint()
+    test_observation = SimpleObservation(collection=test_collection,
+                                         observation_id=test_obs_id,
+                                         algorithm=Algorithm(name='exposure'))
+    test_result = caom2utils.fits2caom2._augment(
+        obs=test_observation,
+        product_id=test_product_id,
+        uri=test_uri,
+        blueprint=test_blueprint,
+        subject=net.Subject(),
+        external_url='https://localhost/files/test_file.fits.gz',
+    )
+    assert test_result is not None, 'expect a result'
+    assert len(test_result.planes.values()) == 1, 'plane added to result'
+    test_plane = test_result.planes[test_product_id]
+    assert len(test_plane.artifacts.values()) == 1, 'artifact added to plane'
+    assert test_uri in test_plane.artifacts.keys(), 'wrong artifact uri'
+
+
 def test_apply_blueprint():
     # test a Gemini case where there are two keywords, one each for
     # different instruments, and the default ends up getting set when the
@@ -1300,6 +1327,29 @@ def test_apply_blueprint():
         test_parser._headers[0]['IMAGESWV'], 'should not be set'
 
 
+def test_apply_blueprint_execute_external():
+    test_module = importlib.import_module(__name__)
+    test_generic_blueprint = ObsBlueprint(module=test_module)
+    test_generic_blueprint.set('Observation.type', '_get_test_obs_type(parameters)')
+
+    # generic parser
+    test_generic_parser = GenericParser(test_generic_blueprint)
+    assert test_generic_parser is not None, \
+        'expect generic construction to complete'
+    assert test_generic_parser._get_from_list('Observation.type', index=0) \
+           == 'generic_parser_value', 'wrong generic plan value'
+
+    # fits parser
+    test_fits_blueprint = ObsBlueprint(module=test_module)
+    test_fits_blueprint.set('Observation.type', '_get_test_obs_type(parameters)')
+    test_fits_parser = FitsParser(src=sample_file_4axes,
+                                  obs_blueprint=test_fits_blueprint)
+    assert test_fits_parser is not None, \
+        'expect fits construction to complete'
+    assert test_fits_parser._get_from_list('Observation.type', index=0) \
+           == 'fits_parser_value', 'wrong fits plan value'
+
+
 def test_update_artifact_meta_errors():
     test_uri = 'gemini:GEMINI/abc.jpg'
     test_artifact = Artifact(uri=test_uri,
@@ -1415,3 +1465,11 @@ def _get_node(uri, limit, force):
     node.props = {'MD5': '5b00b00d4b06aba986c3663d09aa581f',
                   'length': 682560}
     return node
+
+
+def _get_test_obs_type(parameters):
+    assert parameters is not None
+    if parameters.get('header') is None:
+        return 'generic_parser_value'
+    else:
+        return 'fits_parser_value'