Run full metis_wkf instead of only lm

.github/workflows/run_edps.yaml

@@ -70,6 +70,6 @@ jobs:
           export SOF_DATA="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/data"
           export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/sof"
           edps -lw
-          edps -w metis.metis_lm_img_wkf -i $SOF_DATA -c
-          edps -w metis.metis_lm_img_wkf -i $SOF_DATA | tee edps.stdout.txt
+          edps -w metis.metis_wkf -i $SOF_DATA -c
+          edps -w metis.metis_wkf -i $SOF_DATA | tee edps.stdout.txt
           ! grep "'FAILED'" edps.stdout.txt

Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+pytest-ordering = "*"
+
+[requires]
+python_version = "3.12"

metisp/pymetis/src/pymetis/base/impl.py

@@ -33,8 +33,8 @@ class MetisRecipeImpl(ABC):
     Contains central data flow control and also provides abstract methods to be overridden
     by particular pipeline recipe implementations.
     """
-    InputSet: PipelineInputSet = None
-    Product: PipelineProduct = None
+    InputSet: type[PipelineInputSet] = None
+    Product: type[PipelineProduct] = None
 
     # Available parameters are a class variable. This must be present, even if empty.
     parameters = cpl.ui.ParameterList([])
@@ -45,9 +45,9 @@ def __init__(self, recipe: 'MetisRecipe') -> None:
         self.version = recipe.version
         self.parameters = recipe.parameters
 
-        self.inputset = None
-        self.frameset = None
-        self.header = None
+        self.inputset: PipelineInputSet = None
+        self.frameset: cpl.ui.FrameSet | None = None
+        self.header: cpl.core.PropertyList | None = None
         self.products: Dict[str, PipelineProduct] = {}
         self.product_frames = cpl.ui.FrameSet()
 
@@ -67,7 +67,7 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
             self.import_settings(settings)                # Import and process the provided settings dict
             self.inputset = self.InputSet(frameset)       # Create an appropriate InputSet object
             self.inputset.print_debug()
-            self.inputset.validate()                        # Verify that they are valid (maybe with `schema` too?)
+            self.inputset.validate()                      # Verify that they are valid (maybe with `schema` too?)
             products = self.process_images()              # Do all the actual processing
             self.save_products(products)                  # Save the output products
 
@@ -124,14 +124,15 @@ def save_products(self, products: Dict[str, PipelineProduct]) -> None:
         """
         for name, product in products.items():
             Msg.debug(self.__class__.__qualname__,
-                      f"Saving {name}")
+                      f"Saving product {name}")
             product.save()
 
     def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui.FrameSet:
         """
         Gather all the products and build a FrameSet from their frames so that it can be returned from `run`.
         """
-        Msg.debug(self.__class__.__qualname__, f"Building the product frameset")
+        Msg.debug(self.__class__.__qualname__,
+                  f"Building the product frameset")
         return cpl.ui.FrameSet([product.as_frame() for product in products.values()])
 
     def as_dict(self) -> dict[str, Any]:
@@ -164,4 +165,8 @@ def _create_dummy_image():
         Create a dummy image (absolutely no assumptions, just to have something to work with).
         This function should not survive in the future.
         """
-        return cpl.core.Image.load(os.path.expandvars("$SOF_DATA/LINEARITY_2RG.fits"))
+        return cpl.core.Image.load(os.path.expandvars("$SOF_DATA/LINEARITY_2RG.fits"))
+
+    @property
+    def used_frames(self) -> cpl.ui.FrameSet:
+        return self.inputset.used_frames

metisp/pymetis/src/pymetis/base/product.py

@@ -100,14 +100,18 @@ def __str__(self):
 
     def save(self):
         """ Save this Product to a file """
-        Msg.info(self.__class__.__qualname__, f"Saving product file as {self.output_file_name!r}.")
-        Msg.info(self.__class__.__qualname__, str(self.recipe.frameset))
-        # At least one frame must be tagged as RAW, otherwise it *will not* save (rite of passage!)
+        Msg.info(self.__class__.__qualname__,
+                 f"Saving product file as {self.output_file_name!r}:")
+        Msg.info(self.__class__.__qualname__,
+                 f"All frames ({len(self.recipe.frameset)}): {sorted([frame.tag for frame in self.recipe.frameset])}")
+        Msg.info(self.__class__.__qualname__,
+                 f"Used frames ({len(self.recipe.used_frames)}): {sorted([frame.tag for frame in self.recipe.used_frames])}")
+        # At least one frame in the recipe frameset must be tagged as RAW!
+        # Otherwise, it *will not* save (rite of passage)
         cpl.dfs.save_image(
             self.recipe.frameset,       # All frames for the recipe
             self.recipe.parameters,     # The list of input parameters
-            self.recipe.frameset,       # The list of raw and calibration frames actually used
-                                        # (same as all frames, as we always use all the frames)
+            self.recipe.used_frames,    # The list of raw and calibration frames actually used
             self.image,                 # Image to be saved
             self.recipe.name,           # Name of the recipe
             self.properties,            # Properties to be appended
@@ -118,15 +122,16 @@ def save(self):
 
     @property
     def category(self) -> str:
-        """ Return the category of this product
+        """
+        Return the category of this product
 
-        By default, the tag is the same as the category. Feel free to override.
+        By default, the tag is the same as the category. Feel free to override if needed.
         """
         return self.tag
 
     @property
     def output_file_name(self) -> str:
-        """ Form the output file name (the detector part is variable) """
+        """ Form the output file name """
         return f"{self.category}.fits"
 
 
@@ -166,7 +171,7 @@ def __init__(self,
             self.target = target
 
         """
-            At the moment of instantiation, the `target` attribute must be set *somehow*. Either
+            At the moment of instantiation, the `target` attribute must already be set *somehow*. Either
             -   as a class attribute (if it is constant)
             -   from the constructor (if it is determined from the data)
             -   or as a provided property (if it has to be computed dynamically)

metisp/pymetis/src/pymetis/inputs/base.py

@@ -30,7 +30,7 @@ class PipelineInput:
     _title: str = None                      # No universal title makes sense
     _required: bool = True                  # By default, inputs are required to be present
     _tags: Pattern = None                   # No universal tags are provided
-    _group: str = None                      # No sensible default, must be provided explicitly
+    _group: cpl.ui.Frame.FrameGroup = None  # No sensible default, must be provided explicitly
     _detector: str | None = None            # Not specific to a detector until determined otherwise
 
     @property
@@ -122,7 +122,6 @@ def as_dict(self) -> dict[str, Any]:
             'group': self._group.name,
         }
 
-
     def _verify_same_detector_from_header(self) -> None:
         """
         Verification for headers, currently disabled
@@ -155,3 +154,11 @@ def _verify_same_detector_from_header(self) -> None:
             # raise ValueError(f"Darks from more than one detector found: {set(detectors)}!")
             Msg.warning(self.__class__.__qualname__,
                         f"Darks from more than one detector found: {unique}!")
+
+    @abstractmethod
+    def valid_frames(self) -> cpl.ui.FrameSet:
+        """
+        Return a FrameSet containing all valid, used frames.
+        This is abstract as it differes significantly for Single and Multiple Inputs.
+        """
+        pass

metisp/pymetis/src/pymetis/inputs/common.py

@@ -103,5 +103,5 @@ class WavecalInput(SinglePipelineInput):
 
 class PinholeTableInput(SinglePipelineInput):
     _title: str = "pinhole table"
-    _tags: Pattern = re.compile(r"WCU_PINHOLE_TABLE")
+    _tags: Pattern = re.compile(r"PINHOLE_TABLE")
     _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB

metisp/pymetis/src/pymetis/inputs/inputset.py

@@ -21,10 +21,15 @@
 import operator
 
 from abc import ABCMeta
+from typing import Any
 
 import cpl
 from cpl.core import Msg
 
+from pymetis.inputs.base import PipelineInput
+from pymetis.inputs.single import SinglePipelineInput
+from pymetis.inputs.multiple import MultiplePipelineInput
+
 
 class PipelineInputSet(metaclass=ABCMeta):
     """
@@ -47,8 +52,9 @@ def __init__(self, frameset: cpl.ui.FrameSet, **kwargs):
             Filter the input frameset, capture frames that match criteria and assign them to your own attributes.
             By default, there is nothing: no inputs, no tag_parameters.
         """
-        self.inputs = []            # A list of all inputs for this InputSet.
-        self.tag_parameters = {}    # A set of all tunable parameters determined from tags
+        self.inputs: set[PipelineInput] = set()         # A set of all inputs for this InputSet.
+        self.tag_parameters: dict[str, str] = {}        # A dict of all tunable parameters determined from tags
+        self.frameset = frameset
 
     def validate(self) -> None:
         Msg.debug(self.__class__.__qualname__, f"Validating the inputset {self.inputs}")
@@ -79,15 +85,26 @@ def validate_detectors(self) -> None:
             raise ValueError(f"More than one detector found in inputset: {detectors}")
 
     def print_debug(self, *, offset: int = 0) -> None:
-        Msg.debug(self.__class__.__qualname__, f"{' ' * offset} -- Detailed class info ---")
+        Msg.debug(self.__class__.__qualname__, f"{' ' * offset}--- Detailed class info ---")
         Msg.debug(self.__class__.__qualname__, f"{' ' * offset}{len(self.inputs)} inputs:")
         Msg.debug(self.__class__.__qualname__, str(self.inputs))
 
         for inp in self.inputs:
             inp.print_debug(offset=offset + 4)
 
-    def as_dict(self) -> dict[str, type]:
+    def as_dict(self) -> dict[str, Any]:
         return {
             inp.tags: inp.as_dict()
             for inp in self.inputs
         }
+
+    @property
+    def used_frames(self) -> cpl.ui.FrameSet:
+        frameset = cpl.ui.FrameSet()
+
+        for inp in self.inputs:
+            frames = inp.valid_frames()
+            for frame in frames:
+                frameset.append(frame)
+
+        return frameset

metisp/pymetis/src/pymetis/inputs/mixins.py

@@ -19,11 +19,26 @@
 
 import cpl.ui
 
-from pymetis.inputs import PersistenceMapInput, PipelineInputSet
+from pymetis.inputs import PersistenceMapInput, PipelineInputSet, GainMapInput, LinearityInput
 
 
 class PersistenceInputSetMixin(PipelineInputSet):
     def __init__(self, frameset: cpl.ui.FrameSet):
-        self.persistence_map = PersistenceMapInput(frameset)
         super().__init__(frameset)
-        self.inputs += [self.persistence_map]
+
+        self.persistence_map = PersistenceMapInput(frameset, required=False)
+        self.inputs |= {self.persistence_map}
+
+
+class GainMapInputSetMixin(PipelineInputSet):
+    def __init__(self, frameset: cpl.ui.FrameSet):
+        super().__init__(frameset)
+        self.gain_map = GainMapInput(frameset)
+        self.inputs |= {self.gain_map}
+
+
+class LinearityInputSetMixin(PipelineInputSet):
+    def __init__(self, frameset: cpl.ui.FrameSet):
+        super().__init__(frameset)
+        self.linearity = LinearityInput(frameset)
+        self.inputs |= {self.linearity}

metisp/pymetis/src/pymetis/inputs/multiple.py

@@ -17,9 +17,7 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 """
 
-import functools
-import operator
-from typing import Pattern
+from typing import Pattern, Any
 
 import cpl
 
@@ -101,3 +99,11 @@ def _verify_same_detector(self) -> None:
         None:
             None on success
         """
+
+    def as_dict(self) -> dict[str, Any]:
+        return super().as_dict() | {
+            'frame': str(self.frameset),
+        }
+
+    def valid_frames(self) -> cpl.ui.FrameSet:
+        return self.frameset

metisp/pymetis/src/pymetis/inputs/single.py

@@ -17,7 +17,7 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 """
 
-from typing import Pattern
+from typing import Pattern, Any
 
 import cpl
 
@@ -40,6 +40,7 @@ def __init__(self,
         super().__init__(tags=tags, required=required, **kwargs)
 
         self.tag_matches: dict[str, str] = {}
+
         for frame in frameset:
             if match := self.tags.fullmatch(frame.tag):
                 if self.frame is None:
@@ -94,3 +95,15 @@ def _verify_frame_present(self,
         else:
             Msg.debug(self.__class__.__qualname__,
                       f"Found a {self.title} frame {frame.file}")
+
+    def as_dict(self) -> dict[str, Any]:
+        return super().as_dict() | {
+            'frame': str(self.frame),
+        }
+
+    def valid_frames(self) -> cpl.ui.FrameSet:
+        if self.frame is None:
+            # This may happen for non-required inputs
+            return cpl.ui.FrameSet()
+        else:
+            return cpl.ui.FrameSet([self.frame])

metisp/pymetis/src/pymetis/prefab/darkimage.py

@@ -44,4 +44,4 @@ class InputSet(RawImageProcessor.InputSet):
         def __init__(self, frameset: cpl.ui.FrameSet):
             super().__init__(frameset)
             self.master_dark = self.MasterDarkInput(frameset)
-            self.inputs += [self.master_dark]
+            self.inputs |= {self.master_dark}

metisp/pymetis/src/pymetis/prefab/flat.py

@@ -24,36 +24,32 @@
 import cpl
 from cpl.core import Msg
 
-from pymetis.inputs import PipelineInputSet
-from pymetis.inputs.common import RawInput, MasterDarkInput
+from pymetis.inputs import PipelineInputSet, PersistenceMapInput, LinearityInput
+from pymetis.inputs.common import RawInput, MasterDarkInput, BadpixMapInput, GainMapInput
 
 from .darkimage import DarkImageProcessor
 from ..base.product import PipelineProduct
 
 
 class MetisBaseImgFlatImpl(DarkImageProcessor, ABC):
-    class InputSet(PipelineInputSet):
+    class InputSet(DarkImageProcessor.InputSet):
         """
         Base class for Inputs which create flats. Requires a set of raw frames and a master dark.
         """
-        class RawFlatInput(RawInput):
+        MasterDarkInput = MasterDarkInput
+
+        class RawInput(RawInput):
             """
             A subclass of RawInput that is handling the flat image raws.
             """
             _tags = re.compile(r"(?P<band>(LM|N))_FLAT_(?P<target>LAMP|TWILIGHT)_RAW")
 
-        class DarkFlatInput(MasterDarkInput):
-            """
-            Just a plain MasterDarkInput.
-            """
-            pass
-
-        def __init__(self, frameset):
+        def __init__(self, frameset: cpl.ui.FrameSet):
             super().__init__(frameset)
-            self.raw = self.RawFlatInput(frameset)
-            self.master_dark = MasterDarkInput(frameset)
-            self.inputs = [self.raw, self.master_dark]
-
+            #self.persistence = PersistenceMapInput(frameset)
+            #self.linearity = LinearityInput(frameset)
+            #self.gain_map = GainMapInput(frameset)
+            #self.inputs |= {self.persistence, self.linearity, self.gain_map}
 
     class Product(PipelineProduct):
         group = cpl.ui.Frame.FrameGroup.PRODUCT
@@ -83,7 +79,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         # TODO: Detect detector
         # TODO: Twilight
 
-        raw_images = self.load_raw_images()
+        raw_images = self.inputset.load_raw_images()
         master_dark = cpl.core.Image.load(self.inputset.master_dark.frame.file, extension=0)
 
         for raw_image in raw_images:
@@ -96,7 +92,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         # TODO: preprocessing steps like persistence correction / nonlinearity (or not) should come here
 
         header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
-        combined_image = self.combine_images(self.load_raw_images(), method)
+        combined_image = self.combine_images(self.inputset.load_raw_images(), method)
 
         self.products = {
             self.name.upper(): self.Product(self, header, combined_image),