Merge pull request #116 from TAlonglong/feature-handle-missing-tle-fr…

…om-file

Catch KeyError from collector from missing TLE

pytroll_collectors/tests/test_region_collector.py

@@ -64,6 +64,8 @@
 _granule_metadata = {"platform_name": "Metop-C",
                      "sensor": "avhrr"}
 
+_granule_metadata_metop_b = {"platform_name": "Metop-B",
+                             "sensor": "avhrr"}
 
 def granule_metadata(s_min):
     """Return common granule_metadata dictionary."""
@@ -73,6 +75,14 @@ def granule_metadata(s_min):
             "uri": f"file://{s_min:d}"}
 
 
+def granule_metadata_metop_b(s_min):
+    """Return common granule_metadata dictionary."""
+    return {**_granule_metadata_metop_b,
+            "start_time": datetime.datetime(2021, 4, 11, 10, s_min, 0),
+            "end_time": datetime.datetime(2021, 4, 11, 10, s_min+3, 0),
+            "uri": f"file://{s_min:d}"}
+
+
 def harvest_schedules(params, save_basename=None, eum_base_url=None):
     """Use this as fake harvester."""
     return None, None
@@ -198,6 +208,15 @@ def test_collect_check_schedules_custom_method_failed(europe_collector_schedule_
     assert test_string in caplog.text
 
 
+@unittest.mock.patch("pyorbital.tlefile.urlopen", new=_fakeopen)
+def test_collect_missing_tle_from_file(europe_collector, caplog):
+    """Test that granules can be collected, but missing TLE raises and exception"""
+    with caplog.at_level(logging.DEBUG):
+        for s_min in (0, 3, 6, 9, 12, 15, 18):
+            with pytest.raises(KeyError):
+                europe_collector.collect({**granule_metadata_metop_b(s_min)})
+
+
 @unittest.mock.patch("pyorbital.tlefile.urlopen", new=_fakeopen)
 def test_adjust_timeout(europe, caplog):
     """Test timeout adjustment."""

pytroll_collectors/tests/test_triggers.py

@@ -264,3 +264,48 @@ def test_getting_metadata_does_not_involve_multiple_sections(self):
                        'uri': 'somefile_20220512T1544_20220512T1545.data',
                        'key1': "value1"
                        }
+
+    def test_filetrigger_exception(self, caplog):
+        """Test getting the metadata."""
+        from pytroll_collectors.triggers._base import FileTrigger
+        import configparser
+
+        def _collectors(metadata):
+            raise KeyError("Found no TLE entry for 'METOP-B' to simulate")
+        collectors = [_collectors]
+        config = configparser.ConfigParser(interpolation=None)
+        section = "section1"
+        config.add_section(section)
+        config.set(section, "pattern", "{name}_{start_time:%Y%m%dT%H%M}_{end_time:%Y%m%dT%H%M}.data")
+        publisher = None
+        trigger = FileTrigger(collectors, dict(config.items("section1")), publisher, publish_topic=None,
+                              publish_message_after_each_reception=False)
+        trigger._process_metadata({'sensor': 'avhrr', 'platform_name': 'Metop-B',
+                                   'start_time': datetime(2022, 9, 1, 10, 22, 3),
+                                   'orbit_number': '51653',
+                                   'uri': 'AVHRR_C_EUMP_20220901102203_51653_eps_o_amv_l2d.bin',
+                                   'uid': 'AVHRR_C_EUMP_20220901102203_51653_eps_o_amv_l2d.bin',
+                                   'origin': '157.249.16.188:9062',
+                                   'end_time': datetime(2022, 9, 1, 10, 25, 3)})
+        assert "Found no TLE entry for 'METOP-B' to simulate" in caplog.text
+
+    @patch('pytroll_collectors.triggers._base.Trigger.publish_collection')
+    def test_filetrigger_collecting_complete(self, patch_publish_collection):
+        """Test getting the metadata."""
+        from pytroll_collectors.triggers._base import FileTrigger
+        import configparser
+        granule_metadata = {'sensor': 'avhrr'}
+
+        def _collectors(metadata):
+            return True
+        collectors = [_collectors]
+        config = configparser.ConfigParser(interpolation=None)
+        section = "section1"
+        config.add_section(section)
+        config.set(section, "pattern", "{name}_{start_time:%Y%m%dT%H%M}_{end_time:%Y%m%dT%H%M}.data")
+        publisher = None
+        trigger = FileTrigger(collectors, dict(config.items("section1")), publisher, publish_topic=None,
+                              publish_message_after_each_reception=False)
+        trigger._process_metadata(granule_metadata)
+
+        patch_publish_collection.assert_called_once()

pytroll_collectors/triggers/_base.py

@@ -81,9 +81,13 @@ def _process_metadata(self, metadata):
             logger.warning("No metadata")
             return
         for collector in self.collectors:
-            res = collector(metadata.copy())
-            if res:
-                self.publish_collection(res)
+            try:
+                res = collector(metadata.copy())
+            except KeyError as ke:
+                logger.exception("collector failed with: %s ", str(ke))
+            else:
+                if res:
+                    self.publish_collection(res)
 
     def publish_collection(self, metadata):
         """Terminate the gathering."""