diff --git a/.github/workflows/run_edps.yaml b/.github/workflows/run_edps.yaml index c742767..94cf80e 100644 --- a/.github/workflows/run_edps.yaml +++ b/.github/workflows/run_edps.yaml @@ -70,6 +70,6 @@ jobs: export SOF_DATA="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/data" export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/sof" edps -lw - edps -w metis.metis_lm_img_wkf -i $SOF_DATA -c - edps -w metis.metis_lm_img_wkf -i $SOF_DATA | tee edps.stdout.txt + edps -w metis.metis_wkf -i $SOF_DATA -c + edps -w metis.metis_wkf -i $SOF_DATA | tee edps.stdout.txt ! grep "'FAILED'" edps.stdout.txt diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..436fd3d --- /dev/null +++ b/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] +pytest-ordering = "*" + +[requires] +python_version = "3.12" diff --git a/metisp/pymetis/src/pymetis/base/impl.py b/metisp/pymetis/src/pymetis/base/impl.py index 3950e90..238abba 100644 --- a/metisp/pymetis/src/pymetis/base/impl.py +++ b/metisp/pymetis/src/pymetis/base/impl.py @@ -33,8 +33,8 @@ class MetisRecipeImpl(ABC): Contains central data flow control and also provides abstract methods to be overridden by particular pipeline recipe implementations. """ - InputSet: PipelineInputSet = None - Product: PipelineProduct = None + InputSet: type[PipelineInputSet] = None + Product: type[PipelineProduct] = None # Available parameters are a class variable. This must be present, even if empty. parameters = cpl.ui.ParameterList([]) @@ -45,9 +45,9 @@ def __init__(self, recipe: 'MetisRecipe') -> None: self.version = recipe.version self.parameters = recipe.parameters - self.inputset = None - self.frameset = None - self.header = None + self.inputset: PipelineInputSet = None + self.frameset: cpl.ui.FrameSet | None = None + self.header: cpl.core.PropertyList | None = None self.products: Dict[str, PipelineProduct] = {} self.product_frames = cpl.ui.FrameSet() @@ -67,7 +67,7 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra self.import_settings(settings) # Import and process the provided settings dict self.inputset = self.InputSet(frameset) # Create an appropriate InputSet object self.inputset.print_debug() - self.inputset.validate() # Verify that they are valid (maybe with `schema` too?) + self.inputset.validate() # Verify that they are valid (maybe with `schema` too?) products = self.process_images() # Do all the actual processing self.save_products(products) # Save the output products @@ -124,14 +124,15 @@ def save_products(self, products: Dict[str, PipelineProduct]) -> None: """ for name, product in products.items(): Msg.debug(self.__class__.__qualname__, - f"Saving {name}") + f"Saving product {name}") product.save() def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui.FrameSet: """ Gather all the products and build a FrameSet from their frames so that it can be returned from `run`. """ - Msg.debug(self.__class__.__qualname__, f"Building the product frameset") + Msg.debug(self.__class__.__qualname__, + f"Building the product frameset") return cpl.ui.FrameSet([product.as_frame() for product in products.values()]) def as_dict(self) -> dict[str, Any]: @@ -164,4 +165,8 @@ def _create_dummy_image(): Create a dummy image (absolutely no assumptions, just to have something to work with). This function should not survive in the future. """ - return cpl.core.Image.load(os.path.expandvars("$SOF_DATA/LINEARITY_2RG.fits")) \ No newline at end of file + return cpl.core.Image.load(os.path.expandvars("$SOF_DATA/LINEARITY_2RG.fits")) + + @property + def used_frames(self) -> cpl.ui.FrameSet: + return self.inputset.used_frames diff --git a/metisp/pymetis/src/pymetis/base/product.py b/metisp/pymetis/src/pymetis/base/product.py index 23e1f64..835b8a8 100644 --- a/metisp/pymetis/src/pymetis/base/product.py +++ b/metisp/pymetis/src/pymetis/base/product.py @@ -100,14 +100,18 @@ def __str__(self): def save(self): """ Save this Product to a file """ - Msg.info(self.__class__.__qualname__, f"Saving product file as {self.output_file_name!r}.") - Msg.info(self.__class__.__qualname__, str(self.recipe.frameset)) - # At least one frame must be tagged as RAW, otherwise it *will not* save (rite of passage!) + Msg.info(self.__class__.__qualname__, + f"Saving product file as {self.output_file_name!r}:") + Msg.info(self.__class__.__qualname__, + f"All frames ({len(self.recipe.frameset)}): {sorted([frame.tag for frame in self.recipe.frameset])}") + Msg.info(self.__class__.__qualname__, + f"Used frames ({len(self.recipe.used_frames)}): {sorted([frame.tag for frame in self.recipe.used_frames])}") + # At least one frame in the recipe frameset must be tagged as RAW! + # Otherwise, it *will not* save (rite of passage) cpl.dfs.save_image( self.recipe.frameset, # All frames for the recipe self.recipe.parameters, # The list of input parameters - self.recipe.frameset, # The list of raw and calibration frames actually used - # (same as all frames, as we always use all the frames) + self.recipe.used_frames, # The list of raw and calibration frames actually used self.image, # Image to be saved self.recipe.name, # Name of the recipe self.properties, # Properties to be appended @@ -118,15 +122,16 @@ def save(self): @property def category(self) -> str: - """ Return the category of this product + """ + Return the category of this product - By default, the tag is the same as the category. Feel free to override. + By default, the tag is the same as the category. Feel free to override if needed. """ return self.tag @property def output_file_name(self) -> str: - """ Form the output file name (the detector part is variable) """ + """ Form the output file name """ return f"{self.category}.fits" @@ -166,7 +171,7 @@ def __init__(self, self.target = target """ - At the moment of instantiation, the `target` attribute must be set *somehow*. Either + At the moment of instantiation, the `target` attribute must already be set *somehow*. Either - as a class attribute (if it is constant) - from the constructor (if it is determined from the data) - or as a provided property (if it has to be computed dynamically) diff --git a/metisp/pymetis/src/pymetis/inputs/base.py b/metisp/pymetis/src/pymetis/inputs/base.py index b7e25d9..abadf21 100644 --- a/metisp/pymetis/src/pymetis/inputs/base.py +++ b/metisp/pymetis/src/pymetis/inputs/base.py @@ -30,7 +30,7 @@ class PipelineInput: _title: str = None # No universal title makes sense _required: bool = True # By default, inputs are required to be present _tags: Pattern = None # No universal tags are provided - _group: str = None # No sensible default, must be provided explicitly + _group: cpl.ui.Frame.FrameGroup = None # No sensible default, must be provided explicitly _detector: str | None = None # Not specific to a detector until determined otherwise @property @@ -122,7 +122,6 @@ def as_dict(self) -> dict[str, Any]: 'group': self._group.name, } - def _verify_same_detector_from_header(self) -> None: """ Verification for headers, currently disabled @@ -155,3 +154,11 @@ def _verify_same_detector_from_header(self) -> None: # raise ValueError(f"Darks from more than one detector found: {set(detectors)}!") Msg.warning(self.__class__.__qualname__, f"Darks from more than one detector found: {unique}!") + + @abstractmethod + def valid_frames(self) -> cpl.ui.FrameSet: + """ + Return a FrameSet containing all valid, used frames. + This is abstract as it differes significantly for Single and Multiple Inputs. + """ + pass \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/inputs/common.py b/metisp/pymetis/src/pymetis/inputs/common.py index 73493d6..2f38220 100644 --- a/metisp/pymetis/src/pymetis/inputs/common.py +++ b/metisp/pymetis/src/pymetis/inputs/common.py @@ -103,5 +103,5 @@ class WavecalInput(SinglePipelineInput): class PinholeTableInput(SinglePipelineInput): _title: str = "pinhole table" - _tags: Pattern = re.compile(r"WCU_PINHOLE_TABLE") + _tags: Pattern = re.compile(r"PINHOLE_TABLE") _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB diff --git a/metisp/pymetis/src/pymetis/inputs/inputset.py b/metisp/pymetis/src/pymetis/inputs/inputset.py index e38bd81..2079ed4 100644 --- a/metisp/pymetis/src/pymetis/inputs/inputset.py +++ b/metisp/pymetis/src/pymetis/inputs/inputset.py @@ -21,10 +21,15 @@ import operator from abc import ABCMeta +from typing import Any import cpl from cpl.core import Msg +from pymetis.inputs.base import PipelineInput +from pymetis.inputs.single import SinglePipelineInput +from pymetis.inputs.multiple import MultiplePipelineInput + class PipelineInputSet(metaclass=ABCMeta): """ @@ -47,8 +52,9 @@ def __init__(self, frameset: cpl.ui.FrameSet, **kwargs): Filter the input frameset, capture frames that match criteria and assign them to your own attributes. By default, there is nothing: no inputs, no tag_parameters. """ - self.inputs = [] # A list of all inputs for this InputSet. - self.tag_parameters = {} # A set of all tunable parameters determined from tags + self.inputs: set[PipelineInput] = set() # A set of all inputs for this InputSet. + self.tag_parameters: dict[str, str] = {} # A dict of all tunable parameters determined from tags + self.frameset = frameset def validate(self) -> None: Msg.debug(self.__class__.__qualname__, f"Validating the inputset {self.inputs}") @@ -79,15 +85,26 @@ def validate_detectors(self) -> None: raise ValueError(f"More than one detector found in inputset: {detectors}") def print_debug(self, *, offset: int = 0) -> None: - Msg.debug(self.__class__.__qualname__, f"{' ' * offset} -- Detailed class info ---") + Msg.debug(self.__class__.__qualname__, f"{' ' * offset}--- Detailed class info ---") Msg.debug(self.__class__.__qualname__, f"{' ' * offset}{len(self.inputs)} inputs:") Msg.debug(self.__class__.__qualname__, str(self.inputs)) for inp in self.inputs: inp.print_debug(offset=offset + 4) - def as_dict(self) -> dict[str, type]: + def as_dict(self) -> dict[str, Any]: return { inp.tags: inp.as_dict() for inp in self.inputs } + + @property + def used_frames(self) -> cpl.ui.FrameSet: + frameset = cpl.ui.FrameSet() + + for inp in self.inputs: + frames = inp.valid_frames() + for frame in frames: + frameset.append(frame) + + return frameset diff --git a/metisp/pymetis/src/pymetis/inputs/mixins.py b/metisp/pymetis/src/pymetis/inputs/mixins.py index 6db29d7..359f14e 100644 --- a/metisp/pymetis/src/pymetis/inputs/mixins.py +++ b/metisp/pymetis/src/pymetis/inputs/mixins.py @@ -19,11 +19,26 @@ import cpl.ui -from pymetis.inputs import PersistenceMapInput, PipelineInputSet +from pymetis.inputs import PersistenceMapInput, PipelineInputSet, GainMapInput, LinearityInput class PersistenceInputSetMixin(PipelineInputSet): def __init__(self, frameset: cpl.ui.FrameSet): - self.persistence_map = PersistenceMapInput(frameset) super().__init__(frameset) - self.inputs += [self.persistence_map] \ No newline at end of file + + self.persistence_map = PersistenceMapInput(frameset, required=False) + self.inputs |= {self.persistence_map} + + +class GainMapInputSetMixin(PipelineInputSet): + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + self.gain_map = GainMapInput(frameset) + self.inputs |= {self.gain_map} + + +class LinearityInputSetMixin(PipelineInputSet): + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + self.linearity = LinearityInput(frameset) + self.inputs |= {self.linearity} \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/inputs/multiple.py b/metisp/pymetis/src/pymetis/inputs/multiple.py index 89a4415..0bff0db 100644 --- a/metisp/pymetis/src/pymetis/inputs/multiple.py +++ b/metisp/pymetis/src/pymetis/inputs/multiple.py @@ -17,9 +17,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """ -import functools -import operator -from typing import Pattern +from typing import Pattern, Any import cpl @@ -101,3 +99,11 @@ def _verify_same_detector(self) -> None: None: None on success """ + + def as_dict(self) -> dict[str, Any]: + return super().as_dict() | { + 'frame': str(self.frameset), + } + + def valid_frames(self) -> cpl.ui.FrameSet: + return self.frameset \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/inputs/single.py b/metisp/pymetis/src/pymetis/inputs/single.py index 8fccf51..7f0d700 100644 --- a/metisp/pymetis/src/pymetis/inputs/single.py +++ b/metisp/pymetis/src/pymetis/inputs/single.py @@ -17,7 +17,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """ -from typing import Pattern +from typing import Pattern, Any import cpl @@ -40,6 +40,7 @@ def __init__(self, super().__init__(tags=tags, required=required, **kwargs) self.tag_matches: dict[str, str] = {} + for frame in frameset: if match := self.tags.fullmatch(frame.tag): if self.frame is None: @@ -94,3 +95,15 @@ def _verify_frame_present(self, else: Msg.debug(self.__class__.__qualname__, f"Found a {self.title} frame {frame.file}") + + def as_dict(self) -> dict[str, Any]: + return super().as_dict() | { + 'frame': str(self.frame), + } + + def valid_frames(self) -> cpl.ui.FrameSet: + if self.frame is None: + # This may happen for non-required inputs + return cpl.ui.FrameSet() + else: + return cpl.ui.FrameSet([self.frame]) \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/prefab/darkimage.py b/metisp/pymetis/src/pymetis/prefab/darkimage.py index 4a8a56e..e8f625c 100644 --- a/metisp/pymetis/src/pymetis/prefab/darkimage.py +++ b/metisp/pymetis/src/pymetis/prefab/darkimage.py @@ -44,4 +44,4 @@ class InputSet(RawImageProcessor.InputSet): def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) self.master_dark = self.MasterDarkInput(frameset) - self.inputs += [self.master_dark] + self.inputs |= {self.master_dark} diff --git a/metisp/pymetis/src/pymetis/prefab/flat.py b/metisp/pymetis/src/pymetis/prefab/flat.py index 64aaa5c..1079a8f 100644 --- a/metisp/pymetis/src/pymetis/prefab/flat.py +++ b/metisp/pymetis/src/pymetis/prefab/flat.py @@ -24,36 +24,32 @@ import cpl from cpl.core import Msg -from pymetis.inputs import PipelineInputSet -from pymetis.inputs.common import RawInput, MasterDarkInput +from pymetis.inputs import PipelineInputSet, PersistenceMapInput, LinearityInput +from pymetis.inputs.common import RawInput, MasterDarkInput, BadpixMapInput, GainMapInput from .darkimage import DarkImageProcessor from ..base.product import PipelineProduct class MetisBaseImgFlatImpl(DarkImageProcessor, ABC): - class InputSet(PipelineInputSet): + class InputSet(DarkImageProcessor.InputSet): """ Base class for Inputs which create flats. Requires a set of raw frames and a master dark. """ - class RawFlatInput(RawInput): + MasterDarkInput = MasterDarkInput + + class RawInput(RawInput): """ A subclass of RawInput that is handling the flat image raws. """ _tags = re.compile(r"(?P(LM|N))_FLAT_(?PLAMP|TWILIGHT)_RAW") - class DarkFlatInput(MasterDarkInput): - """ - Just a plain MasterDarkInput. - """ - pass - - def __init__(self, frameset): + def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) - self.raw = self.RawFlatInput(frameset) - self.master_dark = MasterDarkInput(frameset) - self.inputs = [self.raw, self.master_dark] - + #self.persistence = PersistenceMapInput(frameset) + #self.linearity = LinearityInput(frameset) + #self.gain_map = GainMapInput(frameset) + #self.inputs |= {self.persistence, self.linearity, self.gain_map} class Product(PipelineProduct): group = cpl.ui.Frame.FrameGroup.PRODUCT @@ -83,7 +79,7 @@ def process_images(self) -> Dict[str, PipelineProduct]: # TODO: Detect detector # TODO: Twilight - raw_images = self.load_raw_images() + raw_images = self.inputset.load_raw_images() master_dark = cpl.core.Image.load(self.inputset.master_dark.frame.file, extension=0) for raw_image in raw_images: @@ -96,7 +92,7 @@ def process_images(self) -> Dict[str, PipelineProduct]: # TODO: preprocessing steps like persistence correction / nonlinearity (or not) should come here header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0) - combined_image = self.combine_images(self.load_raw_images(), method) + combined_image = self.combine_images(self.inputset.load_raw_images(), method) self.products = { self.name.upper(): self.Product(self, header, combined_image), diff --git a/metisp/pymetis/src/pymetis/prefab/rawimage.py b/metisp/pymetis/src/pymetis/prefab/rawimage.py index ca3841b..0f4b35f 100644 --- a/metisp/pymetis/src/pymetis/prefab/rawimage.py +++ b/metisp/pymetis/src/pymetis/prefab/rawimage.py @@ -40,34 +40,33 @@ class InputSet(PipelineInputSet): def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) self.raw = self.RawInput(frameset) - self.inputs += [self.raw] + self.inputs |= {self.raw} - def load_raw_images(self) -> cpl.core.ImageList: - """ - Always load a set of raw images, as determined by the tags. - Chi-Hung has warned Martin that this is unnecessary and fills the memory quickly, - but if we are to use CPL functions, Martin does not see a way around it. - """ - output = cpl.core.ImageList() - - for idx, frame in enumerate(self.inputset.raw.frameset): - Msg.info(self.__class__.__qualname__, f"Processing input frame #{idx}: {frame.file!r}...") + def load_raw_images(self) -> cpl.core.ImageList: + """ + Always load a set of raw images, as determined by the tags. + Chi-Hung has warned Martin that this is unnecessary and fills the memory quickly, + but if we are to use CPL functions, Martin does not see a way around it. + """ + output = cpl.core.ImageList() - # Append the loaded image to an image list - Msg.debug(self.__class__.__qualname__, f"Loading input image {frame.file}") - output.append(cpl.core.Image.load(frame.file, extension=1)) + for idx, frame in enumerate(self.raw.frameset): + Msg.info(self.__class__.__qualname__, + f"Processing input frame #{idx}: {frame.file!r}...") + output.append(cpl.core.Image.load(frame.file, extension=1)) - return output + return output @classmethod def combine_images(cls, images: cpl.core.ImageList, - method: Literal['add'] | Literal['average'] | Literal['median']): + method: Literal['add'] | Literal['average'] | Literal['median']) -> cpl.core.Image: """ Basic helper method to combine images using one of `add`, `average` or `median`. Probably not a universal panacea, but it recurs often enough to warrant being here. """ - Msg.info(cls.__qualname__, f"Combining images using method {method!r}") + Msg.info(cls.__qualname__, + f"Combining {len(images)} images using method {method!r}") combined_image = None match method: case "add": diff --git a/metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py b/metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py index 1486844..6acec33 100644 --- a/metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py +++ b/metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py @@ -31,6 +31,7 @@ from pymetis.base.product import PipelineProduct from pymetis.prefab.rawimage import RawImageProcessor + class MetisCalChophomeImpl(RawImageProcessor): # TODO replace parent class? """Implementation class for metis_cal_chophome""" target = "LM_CHOPHOME" @@ -50,11 +51,11 @@ def __init__(self, frameset: cpl.ui.FrameSet): self.linearity = LinearityInput(frameset) self.gain_map = GainMapInput(frameset) self.persistence = PersistenceMapInput(frameset, required=False) - self.badpixmap = BadpixMapInput(frameset, required=False) + self.badpix_map = BadpixMapInput(frameset, required=False) self.pinhole_table = PinholeTableInput(frameset, required=True) - self.inputs += [self.background, self.linearity, self.gain_map, - self.badpixmap, self.persistence] + self.inputs |= {self.background, self.linearity, self.gain_map, + self.badpix_map, self.persistence, self.pinhole_table} class ProductCombined(PipelineProduct): diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_calibrate.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_calibrate.py index 9a4e282..56b3293 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_calibrate.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_calibrate.py @@ -50,7 +50,7 @@ def __init__(self, frameset: cpl.ui.FrameSet): self.telluric = self.TelluricInput(frameset) self.fluxcal = self.FluxcalTabInput(frameset) - self.inputs += [self.sci_reduced, self.telluric, self.fluxcal] + self.inputs |= {self.sci_reduced, self.telluric, self.fluxcal} class ProductSciCubeCalibrated(PipelineProduct): tag = rf"IFU_SCI_CUBE_CALIBRATED" diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_distortion.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_distortion.py index 2916b93..4da89d9 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_distortion.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_distortion.py @@ -25,27 +25,25 @@ from pymetis.base.recipe import MetisRecipe from pymetis.base.product import PipelineProduct -from pymetis.inputs import RawInput, SinglePipelineInput +from pymetis.inputs import RawInput, SinglePipelineInput, MasterDarkInput +from pymetis.inputs.common import PinholeTableInput +from pymetis.inputs.mixins import PersistenceInputSetMixin, LinearityInputSetMixin, GainMapInputSetMixin +from pymetis.prefab.darkimage import DarkImageProcessor from pymetis.prefab.rawimage import RawImageProcessor -class MetisIfuDistortionImpl(RawImageProcessor): - class InputSet(RawImageProcessor.InputSet): +class MetisIfuDistortionImpl(DarkImageProcessor): + class InputSet(LinearityInputSetMixin, GainMapInputSetMixin, PersistenceInputSetMixin, DarkImageProcessor.InputSet): + MasterDarkInput = MasterDarkInput + class RawInput(RawInput): _tags = re.compile(r"IFU_DISTORTION_RAW") - class PinholeTableInput(SinglePipelineInput): - _tags = re.compile(r"PINHOLE_TABLE") - _title = "pinhole table" - _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB - def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) - self.pinhole_table = SinglePipelineInput(frameset, - tags=re.compile(r"PINHOLE_TABLE"), - title="pinhole table", - group=cpl.ui.Frame.FrameGroup.CALIB) - self.inputs += [self.pinhole_table] + self.pinhole_table = PinholeTableInput(frameset) + + self.inputs |= {self.pinhole_table} class ProductIfuDistortionTable(PipelineProduct): diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_postprocess.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_postprocess.py index 0e43437..d8c463f 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_postprocess.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_postprocess.py @@ -38,7 +38,7 @@ class SciCubeCalibratedInput(SinglePipelineInput): def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) self.sci_cube_calibrated = self.SciCubeCalibratedInput(frameset) - self.inputs += [self.sci_cube_calibrated] + self.inputs |= {self.sci_cube_calibrated} class ProductSciCoadd(PipelineProduct): level = cpl.ui.Frame.FrameLevel.FINAL diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_reduce.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_reduce.py index 87f7d9d..d16fd1b 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_reduce.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_reduce.py @@ -26,6 +26,7 @@ from pymetis.base.product import PipelineProduct, TargetSpecificProduct from pymetis.inputs import SinglePipelineInput from pymetis.inputs.common import RawInput, MasterDarkInput, LinearityInput, PersistenceMapInput +from pymetis.inputs.mixins import PersistenceInputSetMixin, GainMapInputSetMixin from pymetis.prefab.darkimage import DarkImageProcessor @@ -33,12 +34,16 @@ class MetisIfuReduceImpl(DarkImageProcessor): target: Literal["SCI"] | Literal["STD"] = None - class InputSet(DarkImageProcessor.InputSet): + class InputSet(GainMapInputSetMixin, PersistenceInputSetMixin, DarkImageProcessor.InputSet): detector = "IFU" class RawInput(RawInput): _tags = re.compile(r"IFU_(?PSCI|STD)_RAW") + class RawSkyInput(RawInput): + _tags = re.compile(r"IFU_SKY_RAW") + _title = "blank sky image" + class MasterDarkInput(MasterDarkInput): _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.RAW @@ -52,18 +57,24 @@ class DistortionTableInput(SinglePipelineInput): _group = cpl.ui.Frame.FrameGroup.CALIB _title = "Distortion table" + class RsrfInput(SinglePipelineInput): + _tags = re.compile(r"RSRF_IFU") + _group = cpl.ui.Frame.FrameGroup.CALIB + _title = "RSRF" + def __init__(self, frameset: cpl.ui.FrameSet): """ Here we also define all input frames specific for this recipe, except those handled by mixins. """ super().__init__(frameset) self.raw = self.RawInput(frameset) + self.sky = self.RawSkyInput(frameset) self.linearity_map = LinearityInput(frameset) - self.persistence_map = PersistenceMapInput(frameset) self.master_dark = self.MasterDarkInput(frameset) self.ifu_wavecal = self.WavecalInput(frameset) + self.rsrf = self.RsrfInput(frameset) self.ifu_distortion_table = self.DistortionTableInput(frameset) - self.inputs += [self.linearity_map, self.persistence_map, self.master_dark, self.ifu_wavecal, self.ifu_distortion_table] + self.inputs |= {self.sky, self.linearity_map, self.rsrf, self.ifu_wavecal, self.ifu_distortion_table} class ProductReduced(TargetSpecificProduct): level = cpl.ui.Frame.FrameLevel.FINAL @@ -104,7 +115,7 @@ def process_images(self) -> Dict[str, PipelineProduct]: self.target = self.inputset.tag_parameters["target"] header = cpl.core.PropertyList() - images = self.load_raw_images() + images = self.inputset.load_raw_images() image = self.combine_images(images, "add") self.products = { diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_rsrf.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_rsrf.py index 976880a..18c52c5 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_rsrf.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_rsrf.py @@ -23,21 +23,16 @@ from pymetis.base import MetisRecipe from pymetis.base.product import PipelineProduct -from pymetis.inputs import SinglePipelineInput, MultiplePipelineInput, BadpixMapInput, \ - MasterDarkInput, LinearityInput, RawInput, GainMapInput +from pymetis.inputs.common import SinglePipelineInput, MultiplePipelineInput, \ + BadpixMapInput, MasterDarkInput, LinearityInput, \ + RawInput, GainMapInput, PersistenceMapInput, \ + WavecalInput, DistortionTableInput from pymetis.inputs.mixins import PersistenceInputSetMixin from pymetis.prefab.darkimage import DarkImageProcessor -class RsrfMasterDarkInput(MasterDarkInput): - pass -class DistortionTableInput(SinglePipelineInput): - _tags = re.compile(r"IFU_DISTORTION_TABLE") - _title = "distortion table" - _group = cpl.ui.Frame.FrameGroup.CALIB - class MetisIfuRsrfImpl(DarkImageProcessor): class InputSet(PersistenceInputSetMixin, DarkImageProcessor.InputSet): @@ -47,55 +42,263 @@ class RawInput(RawInput): MasterDarkInput = MasterDarkInput + class RsrfWcuOffInput(RawInput): + """ + WCU_OFF input illuminated by the WCU up-to and including the + integrating sphere, but no source. + """ + _tags = re.compile(r"IFU_WCU_OFF_RAW") + _title = "IFU WCU off" + def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) + self.background = self.RsrfWcuOffInput(frameset) + self.linearity = LinearityInput(frameset) self.gain_map = GainMapInput(frameset) self.distortion_table = DistortionTableInput(frameset) + self.wavecal = WavecalInput(frameset) + self.badpixmap = BadpixMapInput(frameset, required=False) + + self.inputs |= {self.background, self.linearity, + self.gain_map, self.distortion_table, + self.wavecal, self.badpixmap} + + class ProductBackground(PipelineProduct): + """ + Intermediate product: the instrumental background (WCU OFF) + """ + group = cpl.ui.Frame.FrameGroup.PRODUCT # TBC + level = cpl.ui.Frame.FrameLevel.INTERMEDIATE + frame_type = cpl.ui.Frame.FrameType.IMAGE - self.inputs += [self.gain_map, self.distortion_table] + # SKEL: copy product keywords from header + def add_properties(self): + super().add_properties() + self.properties.append(self.header) + + @property + def category(self) -> str: + return "IFU_RSRF_BACKGROUND" + + @property + def output_file_name(self) -> str: + return f"{self.category}.fits" + + @property + def tag(self) -> str: + return rf"{self.category}" class ProductMasterFlatIfu(PipelineProduct): - category = rf"MASTER_FLAT_IFU" - tag = category + group = cpl.ui.Frame.FrameGroup.CALIB # TBC level = cpl.ui.Frame.FrameLevel.FINAL frame_type = cpl.ui.Frame.FrameType.IMAGE + # SKEL: copy product keywords from header + def add_properties(self): + super().add_properties() + self.properties.append(self.header) + + @property + def category(self) -> str: + return rf"MASTER_FLAT_IFU" + + @property + def output_file_name(self) -> str: + return f"{self.category}.fits" + + @property + def tag(self) -> str: + return rf"{self.category}" + + class ProductRsrfIfu(PipelineProduct): - category = rf"RSRF_IFU" - tag = category + group = cpl.ui.Frame.FrameGroup.CALIB # TBC level = cpl.ui.Frame.FrameLevel.FINAL - frame_type = cpl.ui.Frame.FrameType.IMAGE + frame_type = cpl.ui.Frame.FrameType.IMAGE # set of 1D spectra? + + # SKEL: copy product keywords from header + def add_properties(self): + super().add_properties() + self.properties.append(self.header) + + @property + def category(self) -> str: + return rf"RSRF_IFU" + + @property + def output_file_name(self) -> str: + return f"{self.category}.fits" + + @property + def tag(self) -> str: + return rf"{self.category}" class ProductBadpixMapIfu(PipelineProduct): - category = rf"BADPIX_MAP_IFU" - tag = category + group = cpl.ui.Frame.FrameGroup.CALIB # TBC level = cpl.ui.Frame.FrameLevel.FINAL frame_type = cpl.ui.Frame.FrameType.IMAGE + # SKEL: copy product keywords from header + def add_properties(self): + super().add_properties() + self.properties.append(self.header) + + @property + def category(self) -> str: + return rf"BADPIX_MAP_IFU" + + @property + def output_file_name(self) -> str: + return f"{self.category}.fits" + + @property + def tag(self) -> str: + return rf"{self.category}" + def process_images(self) -> Dict[str, PipelineProduct]: - # self.correct_telluric() - # self.apply_fluxcal() + # TODO: FUNC: basic raw processing of RSRF and WCU_OFF input frames: + # - dark subtraction? (subtracting WCU_OFF frame might suffice?) + # - gain / linearity correction? (as for dark subtraction) + # - master dark will be used for bad-pixel map as a minimum + + # create bad pixel map + # TODO: FUNC: create updated bad pixel map + badpix_hdr = cpl.core.PropertyList() + # placeholder data for now - bad-pixel map based on master_dark + badpix_img = cpl.core.Image.load(self.inputset.master_dark.frame.file, + extension=0) + # TODO: create QC1 parameters: + qc_badpix_count = 0 + # SKEL: Add QC keywords + badpix_hdr.append( + cpl.core.Property( + "QC IFU RSRF NBADPIX", + cpl.core.Type.INT, + qc_badpix_count, + ) + ) + + # create master WCU_OFF background image + background_hdr = \ + cpl.core.PropertyList() + # self.inputset.background.frameset.dump() # debug + bg_images = self.load_images(self.inputset.background.frameset) + background_img = self.combine_images(bg_images, "median") # if >2 images + # TODO: SKEL: define usedframes? + # TODO: SKEL: Add product keywords - currently none defined in DRLD + + # create 2D flat images (one for each raw input image?) + spec_flat_hdr = \ + cpl.core.PropertyList() + raw_images = self.load_images(self.inputset.raw.frameset) + raw_images.subtract_image(background_img) + # TODO: FUNC: group RSRF input frames (using EDPS wokflow?) by + # 1. BB temperature + # 2. int_sphere entrance aperture size + # 3. Chopper mirror position + # TODO: FUNC: collapse each group into a 2D image + # TODO: FUNC: apply distortion correction and wavelength calibration + # TODO: FUNC: divide each collapsed image by ideal continuum spec image, + # given T_BB_lamp and normalise + # SKEL: Add QC keywords + spec_flat_hdr.append( + cpl.core.Property( + "QC IFU RSRF NBADPIX", + cpl.core.Type.INT, + qc_badpix_count, + ) + ) + + # SKEL: placeholder single-file, single-extension data product for now + spec_flat_img = self.combine_images(raw_images, "add") - header = cpl.core.PropertyList() - images = self.load_raw_images() - image = self.combine_images(images, "add") + # create 1D RSRF + # TODO: FUNC: average 2D flat in spatial direction for each trace + rsrf_hdr = \ + cpl.core.PropertyList() + # TODO: SKEL: Add product keywords - currently none defined in DRLD + # SKEL: placeholder data for now + # NOTE: rebin() cpl documentation is incorrect - + # ystart, xstart parameters are *1-based*, NOT 0-based + img_height = spec_flat_img.height + rsrf_img = spec_flat_img.rebin(1, 1, img_height, 1) + rsrf_img.divide_scalar(img_height) + # instantiate products self.products = { - product.category: product(self, header, image) - for product in [self.ProductMasterFlatIfu, self.ProductRsrfIfu, self.ProductBadpixMapIfu] + self.ProductBackground.tag: + self.ProductBackground(self, background_hdr, background_img), + self.ProductMasterFlatIfu.tag: + self.ProductMasterFlatIfu(self, spec_flat_hdr, spec_flat_img), + self.ProductRsrfIfu.tag: + self.ProductRsrfIfu(self, rsrf_hdr, rsrf_img), + self.ProductBadpixMapIfu.tag: + self.ProductBadpixMapIfu(self, badpix_hdr, badpix_img), } + return self.products + def load_images(self, frameset: cpl.ui.FrameSet) -> cpl.core.ImageList: + """Load an imagelist from a FrameSet + + This is a temporary implementation that should be generalised to the + entire pipeline package. It uses cpl functions - these should be + replaced with hdrl functions once they become available, in order + to use uncertainties and masks. + """ + output = cpl.core.ImageList() + + for idx, frame in enumerate(frameset): + cpl.core.Msg.info(self.__class__.__qualname__, + f"Processing input frame #{idx}: {frame.file!r}...") + output.append(cpl.core.Image.load(frame.file, extension=1)) + + return output + class MetisIfuRsrf(MetisRecipe): _name = "metis_ifu_rsrf" _version = "0.1" - _author = "Martin Baláž" - _email = "martin.balaz@univie.ac.at" - _synopsis = "Determine the relative spectral response function" - _description = ( - "Currently just a skeleton prototype." - ) + _author = "Janus Brink" + _email = "janus.brink27@gmail.com" + _synopsis = "Determine the relative spectral response function." + _description = """\ + Create relative spectral response function for the IFU detector + + Inputs + IFU_RSRF_RAW: Raw RSRF images [1-n] + IFU_WCU_RAW_OFF: Background images with WCU black-body closed [1-n] + MASTER_DARK_IFU: Master dark frame [optional?] + BADPIX_MAP_IFU: Bad-pixel map for 2RG detector [optional] + PERSISTENCE_MAP: Persistence map [optional] + GAIN_MAP_IFU: Gain map for 2RG detector + LINEARITY_IFU: Linearity map for 2RG detector + IFU_DISTORTION_TABLE: Distortion coefficients for an IFU data set + IFU_WAVECAL: IFU wavelength calibration + + Matched Keywords + DET.DIT + DET.NDIT + DRS.IFU + + Outputs + MASTER_FLAT_IFU: Master flat frame for IFU image data + RSRF_IFU: 1D relative spectral response function + BADPIX_MAP_IFU: Updated bad-pixel map + + Algorithm + Average / median stack WCU_OFF images to create background image + Subtract background image from individual RSRF RAW frames + TBC: subtract master_dark from above frames first? + TBC: apply gain / linearity corrections to above frames? + TBC: obtain bad pixel map from master_dark? + Create continuum image by mapping Planck spectrum at Tlamp to wavelength + image. + Divide exposures by continuum image. + Create master flat (2D RSRF) - TBC one extension per input exposure? + Average in spatial direction to obtain relative response function + (1D RSRF) - TBC multiple FITS extensions with spectral traces? + """ # This should not be here but without it pyesorex crashes parameters = cpl.ui.ParameterList([ diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_telluric.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_telluric.py index c113800..16808eb 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_telluric.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_telluric.py @@ -29,7 +29,6 @@ class ProductTelluric(PipelineProduct): - category = rf"IFU_TELLURIC" level = cpl.ui.Frame.FrameLevel.FINAL frame_type = cpl.ui.Frame.FrameType.IMAGE @@ -43,10 +42,29 @@ class CombinedInput(SinglePipelineInput): _group = cpl.ui.Frame.FrameGroup.CALIB _tags = re.compile(r"IFU_(?PSCI|STD)_COMBINED") + class LsfKernelInput(SinglePipelineInput): + _title = "LSF kernel" + _group = cpl.ui.Frame.FrameGroup.CALIB + _tags = re.compile(r"LSF_KERNEL") + + class AtmProfileInput(SinglePipelineInput): + _title = "atmospheric profile" + _group = cpl.ui.Frame.FrameGroup.CALIB + _tags = re.compile(r"ATM_PROFILE") + + class FluxStdCatalogInput(SinglePipelineInput): + _title = "flux std catalog" + _group = cpl.ui.Frame.FrameGroup.CALIB + _tags = re.compile(r"FLUXSTD_CATALOG") + def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) self.combined = self.CombinedInput(frameset) - self.inputs += [self.combined] + self.lsf_kernel = self.LsfKernelInput(frameset) + self.atmospheric_profile = self.AtmProfileInput(frameset) + self.fluxstd_catalog = self.FluxStdCatalogInput(frameset) + + self.inputs |= {self.combined, self.lsf_kernel, self.atmospheric_profile, self.fluxstd_catalog} class ProductSciReduced1D(ProductTelluric): @@ -66,7 +84,7 @@ def process_images(self) -> Dict[str, PipelineProduct]: image = cpl.core.Image.load(self.inputset.combined.frame.file) self.products = { - product.category: product(self, header, image) + str(product.category): product(self, header, image) for product in [self.ProductSciReduced1D, self.ProductIfuTelluric, self.ProductFluxcalTab] } return self.products diff --git a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_wavecal.py b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_wavecal.py index db06440..3e021b2 100644 --- a/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_wavecal.py +++ b/metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_wavecal.py @@ -23,7 +23,7 @@ from pymetis.base import MetisRecipe from pymetis.base.product import PipelineProduct -from pymetis.inputs.common import MasterDarkInput, GainMapInput, RawInput, DistortionTableInput +from pymetis.inputs.common import MasterDarkInput, GainMapInput, RawInput, DistortionTableInput, LinearityInput from pymetis.inputs.mixins import PersistenceInputSetMixin from pymetis.prefab.darkimage import DarkImageProcessor @@ -39,8 +39,9 @@ def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) self.gain_map = GainMapInput(frameset) self.distortion_table = DistortionTableInput(frameset) + self.linearity = LinearityInput(frameset) - self.inputs += [self.gain_map, self.distortion_table] + self.inputs |= {self.gain_map, self.distortion_table, self.linearity} class ProductIfuWavecal(PipelineProduct): category = rf"IFU_WAVECAL" @@ -53,7 +54,7 @@ def process_images(self) -> Dict[str, PipelineProduct]: # self.apply_fluxcal() header = cpl.core.PropertyList() - images = self.load_raw_images() + images = self.inputset.load_raw_images() image = self.combine_images(images, "add") self.products = { diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py index c41b06c..c74eb9d 100644 --- a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_background.py @@ -39,7 +39,7 @@ def __init__(self, frameset: cpl.ui.FrameSet): self.basic_reduced = self.LmBasicReducedInput(frameset) # We need to register the inputs (just to be able to do `for x in self.inputs:`) - self.inputs += [self.basic_reduced] + self.inputs |= {self.basic_reduced} class ProductBkg(PipelineProduct): tag: str = "LM_{target}_BKG" diff --git a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py index d50410f..41bafc5 100644 --- a/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py +++ b/metisp/pymetis/src/pymetis/recipes/img/metis_lm_img_basic_reduce.py @@ -81,7 +81,7 @@ def __init__(self, frameset: cpl.ui.FrameSet): self.gain_map = GainMapInput(frameset) # We need to register the inputs (just to be able to say `for x in self.inputs:`) - self.inputs += [self.master_flat, self.linearity, self.persistence, self.gain_map] + self.inputs |= {self.master_flat, self.linearity, self.persistence, self.gain_map} class Product(TargetSpecificProduct): """ diff --git a/metisp/pymetis/src/pymetis/recipes/instrument/metis_pupil_imaging.py b/metisp/pymetis/src/pymetis/recipes/instrument/metis_pupil_imaging.py index 4eb8932..f175dae 100644 --- a/metisp/pymetis/src/pymetis/recipes/instrument/metis_pupil_imaging.py +++ b/metisp/pymetis/src/pymetis/recipes/instrument/metis_pupil_imaging.py @@ -24,6 +24,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """ +import re + from typing import Dict import cpl @@ -64,33 +66,31 @@ class InputSet(DarkImageProcessor.InputSet): # RawImageProcessor.InputSet. It already knows that it wants a RawInput and MasterDarkInput class, # but does not know about the tags yet. So here we define tags for the raw input class Raw(RawInput): - _tags = ["LM_PUPIL_RAW"] + _tags = re.compile("LM_PUPIL_RAW") # Also one master flat is required. We use a prefabricated class class MasterFlat(MasterFlatInput): - _tags = ["MASTER_IMG_FLAT_LAMP_LM"] + _tags = re.compile("MASTER_IMG_FLAT_LAMP_LM") # We could define the master dark explicitly too, but we can use a prefabricated class instead. # That already has its tags defined (for master darks it's always "MASTER_DARK_{det}"), so we just define # the detector and band. Those are now available for all Input classes here. # Of course, we could be more explicit and define them directly. - detector: str = '2RG' - band: str = 'LM' RawInput = Raw MasterDarkInput = MasterDarkInput def __init__(self, frameset: cpl.ui.FrameSet): super().__init__(frameset) self.master_flat = self.MasterFlat(frameset, - tags=["MASTER_IMG_FLAT_LAMP_{band}", "MASTER_IMG_FLAT_TWILIGHT_{band}"], + tags=re.compile("MASTER_IMG_FLAT_(?PLAMP|TWILIGHT)_(?PLM|N)"), band="LM", det=self.detector) self.linearity = LinearityInput(frameset, det=self.detector) self.persistence = PersistenceMapInput(frameset, required=False) self.gain_map = GainMapInput(frameset, det=self.detector) # We need to register the inputs (just to be able to do `for x in self.inputs:`) - self.inputs += [self.master_flat, self.linearity, self.persistence, self.gain_map] + self.inputs |= {self.master_flat, self.linearity, self.persistence, self.gain_map} class Product(PipelineProduct): """ @@ -203,19 +203,12 @@ class MetisPupilImaging(MetisRecipe): parameters = cpl.ui.ParameterList([ cpl.ui.ParameterEnum( - name="pupil_imaging.stacking.method", - context="pupil_imaging", + name="metis_pupil_imaging.stacking.method", + context="metis_pupil_imaging", description="Name of the method used to combine the input images", default="add", alternatives=("add", "average", "median"), ), - cpl.ui.ParameterEnum( - name="pupil_imaging.band", - context="pupil_imaging", - description="band to run", - default="lm", - alternatives=("lm", "n",), - ) ]) implementation_class = MetisPupilImagingImpl diff --git a/metisp/pymetis/src/pymetis/recipes/metis_det_dark.py b/metisp/pymetis/src/pymetis/recipes/metis_det_dark.py index a8c839c..0656039 100644 --- a/metisp/pymetis/src/pymetis/recipes/metis_det_dark.py +++ b/metisp/pymetis/src/pymetis/recipes/metis_det_dark.py @@ -43,7 +43,7 @@ def __init__(self, frameset: cpl.ui.FrameSet): self.persistence_map = PersistenceMapInput(frameset, required=False) # But should be self.gain_map = GainMapInput(frameset, required=False) # But should be - self.inputs += [self.linearity, self.badpix_map, self.persistence_map, self.gain_map] + self.inputs |= {self.linearity, self.badpix_map, self.persistence_map, self.gain_map} class Product(DetectorSpecificProduct): group = cpl.ui.Frame.FrameGroup.PRODUCT @@ -80,7 +80,7 @@ def process_images(self) -> Dict[str, PipelineProduct]: Msg.info(self.__class__.__qualname__, f"Combining images using method {method!r}") # TODO: preprocessing steps like persistence correction / nonlinearity (or not) - raw_images = self.load_raw_images() + raw_images = self.inputset.load_raw_images() combined_image = self.combine_images(raw_images, method) header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0) diff --git a/metisp/pymetis/src/pymetis/recipes/metis_det_lingain.py b/metisp/pymetis/src/pymetis/recipes/metis_det_lingain.py index f91de7c..2530425 100644 --- a/metisp/pymetis/src/pymetis/recipes/metis_det_lingain.py +++ b/metisp/pymetis/src/pymetis/recipes/metis_det_lingain.py @@ -26,10 +26,9 @@ from pymetis.base.recipe import MetisRecipe from pymetis.base.product import PipelineProduct, DetectorSpecificProduct -from pymetis.inputs.common import RawInput +from pymetis.inputs.common import RawInput, BadpixMapInput from pymetis.mixins.detector import Detector2rgMixin, DetectorGeoMixin, DetectorIfuMixin from pymetis.prefab.rawimage import RawImageProcessor -from pymetis.prefab.darkimage import DarkImageProcessor class LinGainProduct(DetectorSpecificProduct, ABC): @@ -39,11 +38,22 @@ class LinGainProduct(DetectorSpecificProduct, ABC): frame_type = cpl.ui.Frame.FrameType.IMAGE -class MetisDetLinGainImpl(DarkImageProcessor): +class MetisDetLinGainImpl(RawImageProcessor): class InputSet(RawImageProcessor.InputSet): class RawInput(RawInput): _tags = re.compile(r"DETLIN_(?P2RG|GEO|IFU)_RAW") + #class WcuOffInput(RawInput): + # _title = "WCU off raw" + # _tags = re.compile(r"(?PLM|N|IFU)_WCU_OFF_RAW") + + def __init__(self, frameset: cpl.ui.FrameSet): + super().__init__(frameset) + # self.wcu_off = self.WcuOffInput(frameset) + self.badpix_map = BadpixMapInput(frameset, required=False) + # self.inputs |= {self.badpix_map, self.wcu_off} + self.inputs |= {self.badpix_map} + class ProductGain(LinGainProduct): @property def tag(self) -> str: @@ -60,7 +70,7 @@ def tag(self) -> str: return f"BADPIX_MAP_{self.detector:s}" def process_images(self) -> Dict[str, PipelineProduct]: - raw_images = self.load_raw_images() + raw_images = self.inputset.load_raw_images() combined_image = self.combine_images(raw_images, method=self.parameters["metis_det_lingain.stacking.method"].value) diff --git a/metisp/pymetis/src/pymetis/tests/generic.py b/metisp/pymetis/src/pymetis/tests/generic.py index 37cb960..12ca7aa 100644 --- a/metisp/pymetis/src/pymetis/tests/generic.py +++ b/metisp/pymetis/src/pymetis/tests/generic.py @@ -21,15 +21,16 @@ import os.path import pprint import subprocess +import pytest + from abc import ABC from pathlib import Path import cpl -from pymetis.inputs import PipelineInputSet +from pymetis.inputs import PipelineInputSet, MultiplePipelineInput, PipelineInput from pymetis.base.product import PipelineProduct - root = Path(os.path.expandvars("$SOF_DIR")) @@ -56,33 +57,54 @@ class BaseInputSetTest(ABC): impl = None count = None + @pytest.fixture(autouse=True) + def instance(self, load_frameset, sof): + return self.impl.InputSet(load_frameset(sof)) + def test_is_an_inputset(self): assert issubclass(self.impl.InputSet, PipelineInputSet) def test_is_not_abstract(self): assert not inspect.isabstract(self.impl.InputSet) - def test_can_load_and_verify(self, load_frameset, sof): - instance = self.impl.InputSet(load_frameset(sof)) - assert instance.validate() is None + @staticmethod + def test_has_inputs_and_it_is_a_set(instance): + assert isinstance(instance.inputs, set) + + @staticmethod + def test_all_attrs_are_inputs(instance): + for input in instance.inputs: + assert isinstance(input, PipelineInput) + + def test_can_load_and_verify(self, instance): + assert instance.validate() is None, f"InputSet {instance} did not validate" - def test_all_inputs(self, load_frameset, sof): + def test_all_inputs(self, instance): # We should really be testing a class here, not an instance - instance = self.impl.InputSet(load_frameset(sof)) for inp in instance.inputs: assert inp._group is not None assert isinstance(inp._title, str) class RawInputSetTest(BaseInputSetTest): - def test_raw_input_count(self, load_frameset, sof): - instance = self.impl.InputSet(load_frameset(sof)) + def test_is_raw_input_count_correct(self, instance): assert len(instance.raw.frameset) == self.count + @staticmethod + def test_inputset_has_raw(instance): + assert isinstance(instance.raw, MultiplePipelineInput) + class BaseRecipeTest(ABC): + """ + Integration / regression tests for verifying that the recipe can be run + """ _recipe = None + @pytest.fixture(autouse=True) + def frameset(self, load_frameset, sof): + return cpl.ui.FrameSet(load_frameset(sof)) + @classmethod def _run_pyesorex(cls, name, sof): return subprocess.run(['pyesorex', name, root / sof, '--log-level', 'DEBUG'], @@ -92,27 +114,69 @@ def test_recipe_can_be_instantiated(self): recipe = self._recipe() assert isinstance(recipe, cpl.ui.PyRecipe) - def test_recipe_can_be_run_directly(self, load_frameset, sof): + def test_recipe_can_be_run_directly(self, frameset): instance = self._recipe() - frameset = cpl.ui.FrameSet(load_frameset(sof)) - instance.run(frameset, {}) - #pprint.pprint(instance.implementation.as_dict()) + assert isinstance(instance.run(frameset, {}), cpl.ui.FrameSet) + # pprint.pprint(instance.implementation.as_dict(), width=200) def test_recipe_can_be_run_with_pyesorex(self, name, create_pyesorex): pyesorex = create_pyesorex(self._recipe) - assert isinstance(pyesorex.recipe, cpl.ui.PyRecipe) - assert pyesorex.recipe.name == name + assert isinstance(pyesorex.recipe, cpl.ui.PyRecipe), "Recipe is not a cpl.ui.PyRecipe" + assert pyesorex.recipe.name == name, f"Recipe name {name} does not match the pyesorex name {pyesorex.recipe.name}" - @staticmethod - def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex): - output = __class__._run_pyesorex(name, sof) - assert output.returncode == 0 - assert output.stderr == b"" + def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex): + output = self._run_pyesorex(name, sof) + assert output.returncode == 0, "Pyesorex exited with non-zero return code" + assert output.stderr == b"", "Pyesorex exited with non-empty stderr" + + #@pytest.mark.skip(reason="not all recipes have all specified inputs yet") + def test_recipe_uses_all_input_frames(self, frameset): + instance = self._recipe() + instance.run(frameset, {}) + all_frames = sorted([frame.file for frame in instance.implementation.inputset.frameset]) + used_frames = sorted([frame.file for frame in instance.implementation.inputset.used_frames]) + assert all_frames == used_frames,\ + f"Not all frames were used: {instance.implementation.inputset.used_frames!s}" def test_all_parameters_have_correct_context(self): for param in self._recipe.parameters: - assert param.context == self._recipe._name + assert param.context == self._recipe._name,\ + f"Parameter context of {param.name} differs from recipe name {self._recipe.name}" def test_all_parameters_name_starts_with_context(self): for param in self._recipe.parameters: - assert param.name.startswith(self._recipe._name) + assert param.name.startswith(self._recipe._name),\ + f"Parameter name {param.name} does not start with {self._recipe._name}" + + +class BandParamRecipeTest(BaseRecipeTest): + """ + Tests for recipes whose SOFs also specify band parameters ("LM" | "N" | "IFU") + """ + @pytest.mark.parametrize("band", ['lm', 'n', 'ifu']) + def test_recipe_can_be_run_directly(self, load_frameset, band): + sof = f"{self._recipe._name}.{band}.sof" + frameset = load_frameset(sof) + super().test_recipe_can_be_run_directly(frameset) + + @pytest.mark.parametrize("band", ['lm', 'n', 'ifu']) + def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, band, create_pyesorex): + sof = f"{self._recipe._name}.{band}.sof" + super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex) + + +class TargetParamRecipeTest(BaseRecipeTest): + """ + Tests for recipes whose SOFs also specify target parameters ("SCI" | "STD") + """ + @pytest.mark.parametrize("target", ['std', 'sci']) + def test_recipe_can_be_run_directly(self, load_frameset, target): + sof = f"{self._recipe._name}.{target}.sof" + frameset = load_frameset(sof) + super().test_recipe_can_be_run_directly(frameset) + + @pytest.mark.parametrize("target", ['std', 'sci']) + def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, target, create_pyesorex): + sof = f"{self._recipe._name}.{target}.sof" + super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex) + diff --git a/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_reduce.py b/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_reduce.py index 51dd8df..cdfbac7 100644 --- a/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_reduce.py +++ b/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_reduce.py @@ -20,7 +20,7 @@ import pytest from pymetis.recipes.ifu.metis_ifu_reduce import (MetisIfuReduce as Recipe, MetisIfuReduceImpl as Impl) -from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest +from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest, TargetParamRecipeTest @pytest.fixture @@ -33,14 +33,10 @@ def sof(name): return f'{name}.std.sof' -class TestRecipe(BaseRecipeTest): +class TestRecipe(TargetParamRecipeTest): """ A bunch of extremely simple and stupid test cases... just to see if it does something """ _recipe = Recipe - @pytest.mark.parametrize("sof", ["metis_ifu_reduce.std.sof", "metis_ifu_reduce.sci.sof"]) - def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex): - super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex) - class TestInputSet(BaseInputSetTest): impl = Impl diff --git a/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_rsrf.py b/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_rsrf.py new file mode 100644 index 0000000..cf86bf1 --- /dev/null +++ b/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_rsrf.py @@ -0,0 +1,43 @@ +""" +This file is part of the METIS Pipeline. +Copyright (C) 2024 European Southern Observatory + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import pytest + +from pymetis.recipes.ifu.metis_ifu_rsrf import (MetisIfuRsrf as Recipe, MetisIfuRsrfImpl as Impl) +from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest + + +@pytest.fixture +def name(): + return 'metis_ifu_rsrf' + + +@pytest.fixture +def sof(name): + return f'{name}.sof' + + +class TestRecipe(BaseRecipeTest): + """ A bunch of extremely simple and stupid test cases... just to see if it does something """ + _recipe = Recipe + + +class TestInputSet(BaseInputSetTest): + impl = Impl + count = 1 \ No newline at end of file diff --git a/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_telluric.py b/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_telluric.py index 1c38f61..8a53112 100644 --- a/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_telluric.py +++ b/metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_telluric.py @@ -20,7 +20,7 @@ import pytest from pymetis.recipes.ifu.metis_ifu_telluric import (MetisIfuTelluric as Recipe, MetisIfuTelluricImpl as Impl) -from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest +from pymetis.tests.generic import BaseInputSetTest, TargetParamRecipeTest @pytest.fixture @@ -33,7 +33,7 @@ def sof(name): return f'{name}.std.sof' -class TestRecipe(BaseRecipeTest): +class TestRecipe(TargetParamRecipeTest): """ A bunch of extremely simple and stupid test cases... just to see if it does something """ _recipe = Recipe diff --git a/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_flat.py b/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_flat.py index 1e620f1..2b799ed 100644 --- a/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_flat.py +++ b/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_flat.py @@ -19,8 +19,9 @@ import pytest -from pymetis.recipes.img.metis_lm_img_flat import MetisLmImgFlat as Recipe, MetisLmImgFlatImpl as Impl -from generic import BaseInputSetTest, BaseRecipeTest, BaseProductTest +from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest, BaseProductTest +from pymetis.recipes.img.metis_lm_img_flat import (MetisLmImgFlat as Recipe, + MetisLmImgFlatImpl as Impl) @pytest.fixture @@ -30,13 +31,20 @@ def name(): @pytest.fixture def sof(name): - return f"{name}.lamp.sof" + return f'{name}.lamp.sof' class TestRecipe(BaseRecipeTest): - """ A bunch of extremely simple test cases... just to see if it does something """ _recipe = Recipe + @pytest.mark.parametrize("sof", [f"metis_lm_img_flat.{target}.sof" for target in ['lamp', 'twilight']]) + def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex): + super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex) + + @pytest.mark.xfail(reason="files missing from the workflow") + def test_recipe_uses_all_input_frames(self, frameset): + super().test_recipe_uses_all_input_frames(frameset) + class TestInputSet(BaseInputSetTest): impl = Impl diff --git a/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_pupil_imaging.py b/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_pupil_imaging.py index 3c746b0..57a8da6 100644 --- a/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_pupil_imaging.py +++ b/metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_pupil_imaging.py @@ -19,7 +19,7 @@ import pytest -from pymetis.tests.generic import BaseRecipeTest +from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest, BaseProductTest from pymetis.recipes.img.metis_lm_img_basic_reduce import (MetisLmImgBasicReduce as Recipe, MetisLmImgBasicReduceImpl as Impl) @@ -30,10 +30,18 @@ def name(): @pytest.fixture -def sof(): - return 'metis_lm_img_basic_reduce.sof' +def sof(name): + return f'{name}.sof' class TestRecipe(BaseRecipeTest): - """ A bunch of extremely simple and stupid test cases... just to see if it does something """ _recipe = Recipe + + +class TestInputSet(BaseInputSetTest): + impl = Impl + count = 1 + + +class TestProduct(BaseProductTest): + product = Impl.Product diff --git a/metisp/pymetis/src/pymetis/tests/img/test_metis_n_img_flat.py b/metisp/pymetis/src/pymetis/tests/img/test_metis_n_img_flat.py index d12dc42..11c4c43 100644 --- a/metisp/pymetis/src/pymetis/tests/img/test_metis_n_img_flat.py +++ b/metisp/pymetis/src/pymetis/tests/img/test_metis_n_img_flat.py @@ -18,11 +18,10 @@ """ import pytest -import cpl -from pymetis.recipes.img.metis_n_img_flat import MetisNImgFlat as Recipe, MetisNImgFlatImpl as Impl - -from generic import BaseInputSetTest, BaseRecipeTest, BaseProductTest +from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest, BaseProductTest +from pymetis.recipes.img.metis_n_img_flat import (MetisNImgFlat as Recipe, + MetisNImgFlatImpl as Impl) @pytest.fixture @@ -31,18 +30,23 @@ def name(): @pytest.fixture -def sof(): - return 'metis_n_img_flat.lamp.sof' +def sof(name): + return f'{name}.lamp.sof' -# ToDo: GEO raws not yet ready -class DisabledTestRecipe(BaseRecipeTest): - """ A bunch of extremely simple test cases... just to see if it does something """ +class TestRecipe(BaseRecipeTest): _recipe = Recipe + @pytest.mark.parametrize("sof", [f"metis_n_img_flat.{target}.sof" for target in ['lamp', 'twilight']]) + def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex): + super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex) + + @pytest.mark.xfail(reason="files missing from the workflow") + def test_recipe_uses_all_input_frames(self, frameset): + super().test_recipe_uses_all_input_frames(frameset) + -# ToDo: GEO master dark not yet ready -class DisabledTestInputSet(BaseInputSetTest): +class TestInputSet(BaseInputSetTest): impl = Impl count = 1 diff --git a/metisp/pymetis/src/pymetis/tests/instrument/test_metis_pupil_imaging.py b/metisp/pymetis/src/pymetis/tests/instrument/test_metis_pupil_imaging.py new file mode 100644 index 0000000..2d7b89c --- /dev/null +++ b/metisp/pymetis/src/pymetis/tests/instrument/test_metis_pupil_imaging.py @@ -0,0 +1,63 @@ +""" +This file is part of the METIS Pipeline. +Copyright (C) 2024 European Southern Observatory + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import pytest + +from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest +from pymetis.recipes.instrument.metis_pupil_imaging import (MetisPupilImaging as Recipe, + MetisPupilImagingImpl as Impl) + + +@pytest.fixture +def name(): + return 'metis_pupil_imaging' + + +@pytest.fixture +def sof(): + return 'metis_pupil_imaging.lm.sof' + + +class TestRecipe(BaseRecipeTest): + """ A bunch of extremely simple and stupid test cases... just to see if it does something """ + _recipe = Recipe + + @pytest.mark.xfail(reason="SOF file has no master dark yet") + @pytest.mark.parametrize("sof", ["metis_pupil_imaging.lm.sof", "metis_pupil_imaging.n.sof"]) + def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex): + super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex) + + @pytest.mark.xfail(reason="SOF file has no master dark yet") + @pytest.mark.parametrize("sof", ["metis_pupil_imaging.lm.sof", "metis_pupil_imaging.n.sof"]) + def test_recipe_can_be_run_directly(self, load_frameset, sof): + super().test_recipe_can_be_run_directly(load_frameset, sof) + + @pytest.mark.xfail(reason="SOF file has no master dark yet") + @pytest.mark.parametrize("sof", ["metis_pupil_imaging.lm.sof", "metis_pupil_imaging.n.sof"]) + def test_recipe_uses_all_input_frames(self, load_frameset, sof): + super().test_recipe_uses_all_input_frames(load_frameset, sof) + + +class TestInputSet(BaseInputSetTest): + impl = Impl + count = 1 + + @pytest.mark.xfail(reason="SOF file has no master dark yet") + def test_can_load_and_verify(self, load_frameset, sof): + super().test_can_load_and_verify(load_frameset, sof) diff --git a/metisp/pymetis/src/pymetis/tests/test_metis_det_dark.py b/metisp/pymetis/src/pymetis/tests/test_metis_det_dark.py index 0341f6f..869403e 100644 --- a/metisp/pymetis/src/pymetis/tests/test_metis_det_dark.py +++ b/metisp/pymetis/src/pymetis/tests/test_metis_det_dark.py @@ -23,7 +23,7 @@ from pymetis.recipes.metis_det_dark import MetisDetDark as Recipe, MetisDetDarkImpl as Impl -from generic import BaseInputSetTest, BaseRecipeTest, BaseProductTest, RawInputSetTest +from generic import BandParamRecipeTest, BaseProductTest, RawInputSetTest @pytest.fixture @@ -36,21 +36,20 @@ def sof(): return f"metis_det_dark.lm.sof" -class TestRecipe(BaseRecipeTest): +class TestRecipe(BandParamRecipeTest): """ A bunch of simple and stupid test cases... just to see if it does something """ _recipe = Recipe - count = 1 def test_fails_with_files_from_multiple_detectors(self, load_frameset): with pytest.raises(ValueError): instance = self._recipe() - frameset = cpl.ui.FrameSet(load_frameset("incorrect/metis_det_dark.lm.mixed_detectors.sof")) + frameset = cpl.ui.FrameSet(load_frameset("incorrect/metis_det_dark.lm.mixed_raw_detectors.sof")) instance.run(frameset, {}) def test_fails_with_files_from_multiple_detectors_gainmap(self, load_frameset): with pytest.raises(ValueError): instance = self._recipe() - frameset = cpl.ui.FrameSet(load_frameset("incorrect/metis_det_dark.lm.mixed_detectors.again.sof")) + frameset = cpl.ui.FrameSet(load_frameset("incorrect/metis_det_dark.lm.mismatched_detectors.sof")) instance.run(frameset, {}) diff --git a/metisp/pymetis/src/pymetis/tests/test_metis_det_lingain.py b/metisp/pymetis/src/pymetis/tests/test_metis_det_lingain.py index 47d1c23..1a3b25a 100644 --- a/metisp/pymetis/src/pymetis/tests/test_metis_det_lingain.py +++ b/metisp/pymetis/src/pymetis/tests/test_metis_det_lingain.py @@ -21,7 +21,7 @@ from pymetis.recipes.metis_det_lingain import MetisDetLinGain as Recipe, MetisDetLinGainImpl as Impl -from generic import RawInputSetTest, BaseRecipeTest, BaseProductTest +from generic import RawInputSetTest, BandParamRecipeTest, BaseProductTest @pytest.fixture @@ -34,10 +34,14 @@ def sof(): return 'metis_det_lingain.lm.sof' -class TestRecipe(BaseRecipeTest): +class TestRecipe(BandParamRecipeTest): """ A bunch of extremely simple and stupid test cases... just to see if it does something """ _recipe = Recipe + @pytest.mark.xfail(reason="files missing from the workflow") + def test_recipe_uses_all_input_frames(self, frameset): + super().test_recipe_uses_all_input_frames(frameset) + class TestInputSet(RawInputSetTest): impl = Impl diff --git a/metisp/pyrecipes/metis_recipes.py b/metisp/pyrecipes/metis_recipes.py index 0850d11..d7dcd3b 100644 --- a/metisp/pyrecipes/metis_recipes.py +++ b/metisp/pyrecipes/metis_recipes.py @@ -25,7 +25,9 @@ from pymetis.recipes.ifu.metis_ifu_distortion import MetisIfuDistortion from pymetis.recipes.ifu.metis_ifu_calibrate import MetisIfuCalibrate from pymetis.recipes.ifu.metis_ifu_postprocess import MetisIfuPostprocess +from pymetis.recipes.ifu.metis_ifu_rsrf import MetisIfuRsrf from pymetis.recipes.ifu.metis_ifu_reduce import MetisIfuReduce +from pymetis.recipes.ifu.metis_ifu_rsrf import MetisIfuRsrf from pymetis.recipes.ifu.metis_ifu_telluric import MetisIfuTelluric from pymetis.recipes.ifu.metis_ifu_wavecal import MetisIfuWavecal from pymetis.recipes.cal.metis_cal_chophome import MetisCalChophome @@ -40,7 +42,9 @@ MetisIfuDistortion, MetisIfuCalibrate, MetisIfuPostprocess, + MetisIfuRsrf, MetisIfuReduce, + MetisIfuRsrf, MetisIfuTelluric, MetisIfuWavecal, MetisCalChophome,