From 6cb7373a0a05178eadd725d257d67f47ab0d795d Mon Sep 17 00:00:00 2001 From: guillaumegrolleron Date: Tue, 26 Mar 2024 23:20:03 +0100 Subject: [PATCH] Docstring, unit tests for nectarchain tools, component and containers (#114) * docstrings for container module * bugfix : acces to np.array field in container * put the DIRAC import in a with statement to avoid breaking the ctapipe logger * speed up by a factor 2 the event loop by deleteting .tolist() action * cleaning and refactoring of core container class * unit test implementation for the data.container module --------- Co-authored-by: guillaume.grolleron --- .../data/container/chargesContainer.py | 9 + src/nectarchain/data/container/core.py | 226 ++++++++-- src/nectarchain/data/container/eventSource.py | 75 +++- .../data/container/gainContainer.py | 45 ++ .../data/container/tests/test_charge.py | 419 ++++++++++-------- .../data/container/tests/test_core.py | 175 ++++++++ .../data/container/tests/test_gain.py | 227 ++++++++++ .../data/container/tests/test_waveform.py | 215 +++++++++ .../data/container/tests/test_waveforms.py | 110 ----- .../data/container/waveformsContainer.py | 9 + src/nectarchain/data/management.py | 7 +- src/nectarchain/data/tests/test_management.py | 1 + .../makers/component/chargesComponent.py | 12 +- .../makers/component/waveformsComponent.py | 8 +- 14 files changed, 1192 insertions(+), 346 deletions(-) create mode 100644 src/nectarchain/data/container/tests/test_core.py create mode 100644 src/nectarchain/data/container/tests/test_gain.py create mode 100644 src/nectarchain/data/container/tests/test_waveform.py delete mode 100644 src/nectarchain/data/container/tests/test_waveforms.py create mode 100644 src/nectarchain/data/tests/test_management.py diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index 78929e13..02ef5603 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -40,6 +40,15 @@ class ChargesContainer(ArrayDataContainer): class ChargesContainers(TriggerMapContainer): + """ + Class representing a ChargesContainers. + + This class inherits from the `TriggerMapContainer` class and is used to store trigger or slices of data mappings of `ChargesContainer`. + + Attributes: + containers (Field): A field representing the trigger or slices of data mapping of `ChargesContainer`. + """ + containers = Field( default_factory=partial(Map, ChargesContainer), description="trigger or slices of data mapping of ChargesContainer", diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 48fe5b8e..87e41c0d 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -23,19 +23,51 @@ def get_array_keys(container: Container): + """ + Return a list of keys corresponding to fields which are array type in the given container. + + Parameters: + container (Container): The container object to search for array fields. + + Returns: + list: A list of keys corresponding to array fields in the container. + + Example: + >>> container = Container() + >>> container.field1 = np.array([1, 2, 3]) + >>> container.field2 = 5 + >>> container.field3 = np.array([4, 5, 6]) + >>> get_array_keys(container) + ['field1', 'field3'] + """ keys = [] - for field in container.fields: + for key, field in container.fields.items(): if field.type == np.ndarray: - keys.append(field.key) + keys.append(key) return keys class NectarCAMContainer(Container): - """base class for the NectarCAM containers. This contaner cannot berecursive, - to be directly written with a HDF5TableWriter""" + """ + Base class for the NectarCAM containers. This container cannot be recursive, + to be directly written with a HDF5TableWriter. + """ @staticmethod def _container_from_hdf5(path, container_class): + """ + Static method to read a container from an HDF5 file. + + Parameters: + path (str or Path): The path to the HDF5 file. + container_class (Container): The class of the container to be filled with data. + + Yields: + Container: The container from the data in the HDF5 file. + + Example: + >>> container = NectarCAMContainer._container_from_hdf5('path_to_file.h5', MyContainerClass) + """ if isinstance(path, str): path = Path(path) @@ -49,11 +81,30 @@ def _container_from_hdf5(path, container_class): yield container + @classmethod + def from_hdf5(cls, path): + """ + Reads a container from an HDF5 file. + + Parameters: + path (str or Path): The path to the HDF5 file. + + This method will call the _container_from_hdf5 method with the container + argument associated to its own class (ArrayDataContainer) + + Yields: + Container: The container generator linked to the HDF5 file. + + Example: + >>> container = NectarCAMContainer.from_hdf5('path_to_file.h5') + """ + + return cls._container_from_hdf5(path, container_class=cls) + class ArrayDataContainer(NectarCAMContainer): """ A container that holds information about waveforms from a specific run. - Attributes: run_number (int): The run number associated with the waveforms. nevents (int): The number of events. @@ -118,17 +169,91 @@ class ArrayDataContainer(NectarCAMContainer): type=np.ndarray, dtype=np.uint16, ndim=1, description="events multiplicity" ) + +class TriggerMapContainer(Container): + """ + Class representing a TriggerMapContainer. + + This class inherits from the `Container` class and is used to store trigger mappings of containers. + + Attributes: + containers (Field): A field representing the trigger mapping of containers. + + Methods: + is_empty(): Checks if the TriggerMapContainer is empty. + validate(): Validates the TriggerMapContainer by checking if all the containers mapped are filled by correct type. + + Example: + >>> container = TriggerMapContainer() + >>> container.is_empty() + True + >>> container.validate() + None + """ + + containers = Field( + default_factory=partial(Map, Container), + description="trigger mapping of Container", + ) + + @classmethod + def from_hdf5(cls, path, slice_index=None): + """ + Reads a container from an HDF5 file. + + Parameters: + path (str or Path): The path to the HDF5 file. + slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. + + This method will call the _container_from_hdf5 method with the container argument associated to its own class (ArrayDataContainer) + + Yields: + Container: The container generator linked to the HDF5 file. + + Example: + >>> container = ArrayDataContainer.from_hdf5('path_to_file.h5') + """ + + return cls._container_from_hdf5( + path, slice_index=slice_index, container_class=cls + ) + @staticmethod def _container_from_hdf5(path, container_class, slice_index=None): + """ + Reads a container from an HDF5 file. + + Parameters: + path (str or Path): The path to the HDF5 file. + container_class (Container): The class of the container to be read. + slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. + + This method first checks if the path is a string and converts it to a Path object if it is. + It then imports the module of the container class and creates an instance of the container class. + + If the HDF5 file contains more than one slice and no slice index is provided, + it reads all slices and yields a generator of containers. + If a slice index is provided, it reads only the specified slice and returns the container instance. + + Yields: + Container: The container associated to the HDF5 file. + + Raises: + NoSuchNodeError: If the specified node does not exist in the HDF5 file. + Exception: If any other error occurs. + + Example: + >>> container = ArrayDataContainer._container_from_hdf5('path_to_file.h5', MyContainerClass) + """ if isinstance(path, str): path = Path(path) module = importlib.import_module(f"{container_class.__module__}") - container = eval(f"module.{container_class.__name__}s")() + container = eval(f"module.{container_class.__name__}")() with HDF5TableReader(path) as reader: if len(reader._h5file.root.__members__) > 1 and slice_index is None: log.info( - f"reading {container_class.__name__}s containing {len(reader._h5file.root.__members__)} slices, will return a generator" + f"reading {container_class.__name__} containing {len(reader._h5file.root.__members__)} slices, will return a generator" ) for data in reader._h5file.root.__members__: # container.containers[data] = eval(f"module.{container_class.__name__}s")() @@ -163,57 +288,44 @@ def _container_from_hdf5(path, container_class, slice_index=None): else: if slice_index is None: log.info( - f"reading {container_class.__name__}s containing a single slice, will return the {container_class.__name__}s instance" + f"reading {container_class.__name__} containing a single slice, will return the {container_class.__name__} instance" ) data = "data" else: log.info( - f"reading slice {slice_index} of {container_class.__name__}s, will return the {container_class.__name__}s instance" + f"reading slice {slice_index} of {container_class.__name__}, will return the {container_class.__name__} instance" ) data = f"data_{slice_index}" for key, trigger in EventType.__members__.items(): try: - container_data = eval(f"reader._h5file.root.{data}.__members__") - _mask = [ - container_class.__name__ in _word - for _word in container_data - ] - _container_data = np.array(container_data)[_mask] - if len(_container_data) == 1: - tableReader = reader.read( - table_name=f"/{data}/{_container_data[0]}/{trigger.name}", - containers=container_class, - ) - container.containers[trigger] = next(tableReader) - else: - log.info( - f"there is {len(_container_data)} entry corresponding to a {container_class} table save, unable to load" - ) + tableReader = reader.read( + table_name=f"/{data}/{trigger.name}", + containers=eval( + f"module.{container.fields['containers'].default_factory.args[0].__name__}" + ), + ) + container.containers[trigger] = next(tableReader) except NoSuchNodeError as err: log.warning(err) except Exception as err: log.error(err, exc_info=True) raise err yield container - return container - - @classmethod - def from_hdf5(cls, path, slice_index=None): - return cls._container_from_hdf5( - path, slice_index=slice_index, container_class=cls - ) - - -class TriggerMapContainer(Container): - containers = Field( - default_factory=partial(Map, Container), - description="trigger mapping of Container", - ) def is_empty(self): + """This method check if the container is empty + + Returns: + bool: True if the container is empty, False otherwise. + """ return len(self.containers.keys()) == 0 def validate(self): + """apply the validate method recursively to all the containers that are mapped within the TriggerMapContainer + + Raises: + FieldValidationError: if one container is not valid. + """ super().validate() for i, container in enumerate(self.containers): if i == 0: @@ -226,7 +338,43 @@ def validate(self): def merge_map_ArrayDataContainer(triggerMapContainer: TriggerMapContainer): + """ + Merge and map ArrayDataContainer + + This function takes a TriggerMapContainer as input and merges the array fields of the containers mapped within the TriggerMapContainer. The merged array fields are concatenated along the 0th axis. The function also updates the 'nevents' field of the output container by summing the 'nevents' field of all the mapped containers. + + Parameters: + triggerMapContainer (TriggerMapContainer): The TriggerMapContainer object containing the containers to be merged and mapped. + + Returns: + ArrayDataContainer: The merged and mapped ArrayDataContainer object. + + Example: + >>> triggerMapContainer = TriggerMapContainer() + >>> container1 = ArrayDataContainer() + >>> container1.field1 = np.array([1, 2, 3]) + >>> container1.field2 = np.array([4, 5, 6]) + >>> container1.nevents + 3 + >>> container2 = ArrayDataContainer() + >>> container2.field1 = np.array([7, 8, 9]) + >>> container2.field2 = np.array([10, 11, 12]) + >>> container2.nevents + 3 + >>> triggerMapContainer.containers['container1'] = container1 + >>> triggerMapContainer.containers['container2'] = container2 + >>> merged_container = merge_map_ArrayDataContainer(triggerMapContainer) + >>> merged_container.field1 + array([1, 2, 3, 7, 8, 9]) + >>> merged_container.field2 + array([ 4, 5, 6, 10, 11, 12]) + >>> merged_container.nevents + 6 + """ triggerMapContainer.validate() + log.warning( + "TAKE CARE TO MERGE CONTAINERS ONLY IF PIXELS ID, RUN_NUMBER (OR ANY FIELD THAT ARE NOT ARRAY) ARE THE SAME" + ) keys = list(triggerMapContainer.containers.keys()) output_container = copy.deepcopy(triggerMapContainer.containers[keys[0]]) for key in keys[1:]: diff --git a/src/nectarchain/data/container/eventSource.py b/src/nectarchain/data/container/eventSource.py index 0706b1b9..50b24192 100644 --- a/src/nectarchain/data/container/eventSource.py +++ b/src/nectarchain/data/container/eventSource.py @@ -26,16 +26,31 @@ __all__ = ["LightNectarCAMEventSource"] -# DONE event.trigger.event_type -# DONE event.index.event_id -# DONE event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_timestamp -# DONE event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_busy_counter -# DONE event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_event_counter -# DONE event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern -# DONE event.r0.tel[0].waveform - def fill_nectarcam_event_container_from_zfile(self, array_event, event): + """ + Fill the NectarCAM event container from the zfile event data. + + Parameters: + - array_event: The NectarCAMDataContainer object to fill with event data. + - event: The event data from the zfile. + + Returns: + - None + + This function fills the NectarCAM event container in the NectarCAMDataContainer object with the event data from the zfile. It unpacks the necessary data from the event and assigns it to the corresponding fields in the event container. + + The function performs the following steps: + 1. Assigns the tel_id to the local variable. + 2. Creates a new NectarCAMEventContainer object and assigns it to the event_container field of the NectarCAMDataContainer object. + 3. Assigns the extdevices_presence field of the event to the extdevices_presence field of the event_container. + 4. Assigns the counters field of the event to the counters field of the event_container. + 5. Unpacks the TIB data from the event and assigns it to the corresponding fields in the event_container. + 6. Unpacks the CDTS data from the event and assigns it to the corresponding fields in the event_container. + 7. Calls the unpack_feb_data function to unpack the FEB counters and trigger pattern from the event and assign them to the corresponding fields in the event_container. + + """ + tel_id = self.tel_id event_container = NectarCAMEventContainer() array_event.nectarcam.tel[tel_id].evt = event_container @@ -160,6 +175,19 @@ def unpack_feb_data(self, event_container, event): def fill_trigger_info(self, array_event): + """ + Fill the trigger information for a given event. + + Parameters: + array_event (NectarCAMEventContainer): The NectarCAMEventContainer object to fill with trigger information. + + Returns: + None + + Raises: + None + """ + tel_id = self.tel_id nectarcam = array_event.nectarcam.tel[tel_id] tib_available = nectarcam.evt.extdevices_presence & 1 @@ -216,7 +244,38 @@ def fill_trigger_info(self, array_event): class LightNectarCAMEventSource(NectarCAMEventSource): + """ + LightNectarCAMEventSource is a subclass of NectarCAMEventSource that provides a generator for iterating over NectarCAM events. + + This implementation of the NectarCAMEventSource is mucvh lighter than the one within ctapipe_io_nectarcam, only the fileds interesting + for nectachain are kept. + + Attributes: + input_url (str): The input URL of the data source. + max_events (int): The maximum number of events to process. + tel_id (int): The telescope ID. + nectarcam_service (NectarCAMService): The service container for NectarCAM. + trigger_information (bool): Flag indicating whether to fill trigger information in the event container. + obs_ids (list): The list of observation IDs. + multi_file (MultiFileReader): The multi-file reader for reading the data source. + r0_r1_calibrator (R0R1Calibrator): The calibrator for R0 to R1 conversion. + calibrate_flatfields_and_pedestals (bool): Flag indicating whether to calibrate flatfield and pedestal events. + + Methods: + _generator: The generator function that yields NectarCAMDataContainer objects representing each event. + + """ + def _generator(self): + """ + The generator function that yields NectarCAMDataContainer objects representing each event. + + Yields: + NectarCAMDataContainer: The NectarCAMDataContainer object representing each event. + + Raises: + None + """ # container for NectarCAM data array_event = NectarCAMDataContainer() array_event.meta["input_url"] = self.input_url diff --git a/src/nectarchain/data/container/gainContainer.py b/src/nectarchain/data/container/gainContainer.py index 7f2fb36f..d1950cf3 100644 --- a/src/nectarchain/data/container/gainContainer.py +++ b/src/nectarchain/data/container/gainContainer.py @@ -13,6 +13,26 @@ class GainContainer(NectarCAMContainer): + """ + Class representing a GainContainer. + + This class is a subclass of NectarCAMContainer and provides additional fields and methods specific to gain calibration data. + + Attributes: + is_valid (np.ndarray): Array of booleans indicating the validity of each gain value. + high_gain (np.ndarray): Array of high gain values. + low_gain (np.ndarray): Array of low gain values. + pixels_id (np.ndarray): Array of pixel IDs. + + Methods: + from_hdf5(cls, path): Class method to read a GainContainer from an HDF5 file. + Parameters: + path (str or Path): The path to the HDF5 file. + + Yields: + GainContainer: The container from the data in the HDF5 file. + """ + is_valid = Field(type=np.ndarray, dtype=bool, ndim=1, description="is_valid") high_gain = Field( type=np.ndarray, dtype=np.float64, ndim=2, description="high gain" @@ -22,10 +42,35 @@ class GainContainer(NectarCAMContainer): @classmethod def from_hdf5(cls, path): + """Class method to read a GainContainer from an HDF5 file. + + Args: + path (str or Path): The path to the HDF5 file. + + Yields: + GainContainer: The container from the data in the HDF5 file. + """ return super(__class__, cls)._container_from_hdf5(path, container_class=cls) class SPEfitContainer(GainContainer): + """ + Class representing a SPEfitContainer. + + This class is a subclass of GainContainer and provides additional fields specific to single photoelectron (SPE) fit data. + + Attributes: + likelihood (np.ndarray): Array of likelihood values. + p_value (np.ndarray): Array of p-values. + pedestal (np.ndarray): Array of pedestal values. + pedestalWidth (np.ndarray): Array of pedestal widths. + resolution (np.ndarray): Array of resolution values. + luminosity (np.ndarray): Array of luminosity values. + mean (np.ndarray): Array of mean values. + n (np.ndarray): Array of n values. + pp (np.ndarray): Array of pp values. + """ + likelihood = Field( type=np.ndarray, dtype=np.float64, ndim=1, description="likelihood" ) diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index e67f36bb..3d146b7b 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -1,178 +1,247 @@ -# import glob +import glob -# import numpy as np +import numpy as np import pytest +from ctapipe.containers import EventType +from ctapipe.io import HDF5TableWriter -# from nectarchain.data.container import ChargesContainer, ChargesContainerIO -# from nectarchain.makers import ChargesMaker - -pytest.skip( - "Some classes to be imported here were dropped from nectarchain," - "skipping all these tests entirely", - allow_module_level=True, -) - - -# def create_fake_chargeContainer(): -# nevents = TestChargesContainer.nevents -# npixels = TestChargesContainer.npixels -# rng = np.random.default_rng() -# return ChargesContainer( -# pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]), -# nevents=nevents, -# npixels=npixels, -# charges_hg=rng.integers(low=0, high=1000, size=(nevents, npixels)), -# charges_lg=rng.integers(low=0, high=1000, size=(nevents, npixels)), -# peak_hg=rng.integers(low=0, high=60, size=(nevents, npixels)), -# peak_lg=rng.integers(low=0, high=60, size=(nevents, npixels)), -# run_number=TestChargesContainer.run_number, -# camera="TEST", -# broken_pixels_hg=rng.integers(low=0, high=1, size=(nevents, npixels)), -# broken_pixels_lg=rng.integers(low=0, high=1, size=(nevents, npixels)), -# ucts_timestamp=rng.integers(low=0, high=100, size=(nevents)), -# ucts_busy_counter=rng.integers(low=0, high=100, size=(nevents)), -# ucts_event_counter=rng.integers(low=0, high=100, size=(nevents)), -# event_type=rng.integers(low=0, high=1, size=(nevents)), -# event_id=rng.integers(low=0, high=1000, size=(nevents)), -# trig_pattern_all=rng.integers(low=0, high=1, size=(nevents, npixels, 4)), -# trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels)), -# multiplicity=rng.integers(low=0, high=1, size=(nevents)), -# ) - - -# class TestChargesContainer: -# run_number = 1234 -# nevents = 140 -# npixels = 10 -# -# # Tests that a ChargeContainer object can be created with valid input parameters. -# def test_create_charge_container(self): -# pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) -# nevents = TestChargesContainer.nevents -# npixels = TestChargesContainer.npixels -# run_number = TestChargesContainer.run_number -# charges_hg = np.random.randn(nevents, npixels) -# charges_lg = np.random.randn(nevents, npixels) -# peak_hg = np.random.randn(nevents, npixels) -# peak_lg = np.random.randn(nevents, npixels) -# method = "FullWaveformSum" -# charge_container = ChargesContainer( -# charges_hg=charges_hg, -# charges_lg=charges_lg, -# peak_hg=peak_hg, -# peak_lg=peak_lg, -# run_number=run_number, -# pixels_id=pixels_id, -# nevents=nevents, -# npixels=npixels, -# method=method, -# ) -# -# assert np.allclose(charge_container.charges_hg, charges_hg) -# assert np.allclose(charge_container.charges_lg, charges_lg) -# assert np.allclose(charge_container.peak_hg, peak_hg) -# assert np.allclose(charge_container.peak_lg, peak_lg) -# assert charge_container.run_number == run_number -# assert charge_container.pixels_id.tolist() == pixels_id.tolist() -# assert charge_container.nevents == nevents -# assert charge_container.npixels == npixels -# assert charge_container.method == method -# -# # Tests that the from_waveforms method can be called with a valid -# # waveformContainer and method parameter. -# # def test_from_waveforms_valid_input(self): -# # waveform_container = WaveformsContainer(...) -# # method = 'FullWaveformSum' -# # -# # charge_container = ChargeContainer.from_waveforms(waveform_container, method) -# # -# # assert isinstance(charge_container, ChargeContainer) -# -# # Tests that the ChargeContainer object can be written to a file and the file is -# # created. -# def test_write_charge_container(self, tmp_path="/tmp"): -# charge_container = create_fake_chargeContainer() -# tmp_path += f"/{np.random.randn(1)[0]}" -# -# ChargesContainerIO.write(tmp_path, charge_container) -# -# assert ( -# len( -# glob.glob( -# f"{tmp_path}/charge_run{TestChargesContainer.run_number}.fits" -# ) -# ) -# == 1 -# ) -# -# # Tests that a ChargeContainer object can be loaded from a file and the object is -# # correctly initialized. -# def test_load_charge_container(self, tmp_path="/tmp"): -# charge_container = create_fake_chargeContainer() -# tmp_path += f"/{np.random.randn(1)[0]}" -# -# ChargesContainerIO.write(tmp_path, charge_container) -# -# loaded_charge_container = ChargesContainerIO.load( -# tmp_path, TestChargesContainer.run_number -# ) -# -# assert isinstance(loaded_charge_container, ChargesContainer) -# assert np.allclose( -# loaded_charge_container.charges_hg, charge_container.charges_hg -# ) -# assert np.allclose( -# loaded_charge_container.charges_lg, charge_container.charges_lg -# ) -# assert np.allclose(loaded_charge_container.peak_hg, charge_container.peak_hg) -# assert np.allclose(loaded_charge_container.peak_lg, charge_container.peak_lg) -# assert loaded_charge_container.run_number == charge_container.run_number -# assert ( -# loaded_charge_container.pixels_id.tolist() -# == charge_container.pixels_id.tolist() -# ) -# assert loaded_charge_container.nevents == charge_container.nevents -# assert loaded_charge_container.npixels == charge_container.npixels -# assert loaded_charge_container.method == charge_container.method -# -# # Tests that the ChargeContainer object can be sorted by event_id and the object -# # is sorted accordingly. -# def test_sort_charge_container(self): -# charge_container = create_fake_chargeContainer() -# -# sorted_charge_container = ChargesMaker.sort(charge_container) -# -# assert sorted_charge_container.event_id.tolist() == sorted( -# charge_container.event_id.tolist() -# ) -# -# # Tests that the run_number, pixels_id, npixels, nevents, method, multiplicity, -# # and trig_pattern properties of the ChargeContainer object can be accessed and -# # the values are correct. -# def test_access_properties(self): -# pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) -# nevents = 40 -# npixels = 10 -# charges_hg = np.random.randn(nevents, npixels) -# charges_lg = np.random.randn(nevents, npixels) -# peak_hg = np.random.randn(nevents, npixels) -# peak_lg = np.random.randn(nevents, npixels) -# run_number = 1234 -# method = "FullWaveformSum" -# charge_container = ChargesContainer( -# charges_hg=charges_hg, -# charges_lg=charges_lg, -# peak_hg=peak_hg, -# peak_lg=peak_lg, -# run_number=run_number, -# pixels_id=pixels_id, -# nevents=nevents, -# npixels=npixels, -# method=method, -# ) -# -# assert charge_container.run_number == run_number -# assert charge_container.pixels_id.tolist() == pixels_id.tolist() -# assert charge_container.npixels == npixels -# assert charge_container.nevents == nevents -# assert charge_container.method == method +from nectarchain.data.container import ChargesContainer, ChargesContainers + + +def create_fake_chargeContainer(): + nevents = TestChargesContainer.nevents + npixels = TestChargesContainer.npixels + rng = np.random.default_rng() + charge = ChargesContainer( + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16), + nevents=np.uint64(nevents), + npixels=np.uint16(npixels), + charges_hg=rng.integers( + low=0, high=1000, size=(nevents, npixels), dtype=np.uint16 + ), + charges_lg=rng.integers( + low=0, high=1000, size=(nevents, npixels), dtype=np.uint16 + ), + peak_hg=rng.integers(low=0, high=60, size=(nevents, npixels), dtype=np.uint16), + peak_lg=rng.integers(low=0, high=60, size=(nevents, npixels), dtype=np.uint16), + run_number=np.uint16(TestChargesContainer.run_number), + camera="TEST", + method="test_method", + broken_pixels_hg=rng.integers( + low=0, high=1.1, size=(nevents, npixels), dtype=bool + ), + broken_pixels_lg=rng.integers( + low=0, high=1.1, size=(nevents, npixels), dtype=bool + ), + ucts_timestamp=rng.integers(low=0, high=100, size=(nevents), dtype=np.uint64), + ucts_busy_counter=rng.integers( + low=0, high=100, size=(nevents), dtype=np.uint32 + ), + ucts_event_counter=rng.integers( + low=0, high=100, size=(nevents), dtype=np.uint32 + ), + event_type=rng.integers(low=0, high=1, size=(nevents), dtype=np.uint8), + event_id=rng.integers(low=0, high=1000, size=(nevents), dtype=np.uint32), + trig_pattern_all=rng.integers( + low=0, high=1, size=(nevents, npixels, 4), dtype=bool + ), + trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels), dtype=bool), + multiplicity=rng.integers(low=0, high=1, size=(nevents), dtype=np.uint16), + ) + charge.validate() + return charge + + +def create_fake_chargeContainers(): + charge_1 = create_fake_chargeContainer() + charge_2 = create_fake_chargeContainer() + charge = ChargesContainers() + charge.containers[EventType.FLATFIELD] = charge_1 + charge.containers[EventType.SKY_PEDESTAL] = charge_2 + return charge + + +class TestChargesContainer: + run_number = 1234 + nevents = 140 + npixels = 10 + + # Tests that a ChargeContainer object can be created with valid input parameters. + def test_create_charge_container(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16) + nevents = np.uint64(TestChargesContainer.nevents) + npixels = np.uint16(TestChargesContainer.npixels) + run_number = np.uint16(TestChargesContainer.run_number) + charges_hg = np.uint16(np.random.randn(nevents, npixels)) + charges_lg = np.uint16(np.random.randn(nevents, npixels)) + peak_hg = np.uint16(np.random.randn(nevents, npixels)) + peak_lg = np.uint16(np.random.randn(nevents, npixels)) + method = "FullWaveformSum" + charge_container = ChargesContainer( + charges_hg=charges_hg, + charges_lg=charges_lg, + peak_hg=peak_hg, + peak_lg=peak_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + method=method, + ) + charge_container.validate() + assert np.allclose(charge_container.charges_hg, charges_hg) + assert np.allclose(charge_container.charges_lg, charges_lg) + assert np.allclose(charge_container.peak_hg, peak_hg) + assert np.allclose(charge_container.peak_lg, peak_lg) + assert charge_container.run_number == run_number + assert charge_container.pixels_id.tolist() == pixels_id.tolist() + assert charge_container.nevents == nevents + assert charge_container.npixels == npixels + assert charge_container.method == method + + # Tests that the from_waveforms method can be called with a valid + # waveformContainer and method parameter. + # def test_from_waveforms_valid_input(self): + # waveform_container = WaveformsContainer(...) + # method = 'FullWaveformSum' + # + # charge_container = ChargeContainer.from_waveforms(waveform_container, method) + # + # assert isinstance(charge_container, ChargeContainer) + # Tests that the ChargeContainer object can be written to a file and the file is + # created. + def test_write_charge_container(self, tmp_path="/tmp"): + charge_container = create_fake_chargeContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="ChargesContainer", containers=charge_container) + writer.close() + + # Tests that a ChargeContainer object can be loaded from a file and the object is + # correctly initialized. + def test_from_hdf5(self, tmp_path="/tmp"): + charge_container = create_fake_chargeContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="ChargesContainer", containers=charge_container) + writer.close() + + loaded_charge_container = next(ChargesContainer.from_hdf5(tmp_path)) + + assert isinstance(loaded_charge_container, ChargesContainer) + assert np.allclose( + loaded_charge_container.charges_hg, charge_container.charges_hg + ) + assert np.allclose( + loaded_charge_container.charges_lg, charge_container.charges_lg + ) + assert np.allclose(loaded_charge_container.peak_hg, charge_container.peak_hg) + assert np.allclose(loaded_charge_container.peak_lg, charge_container.peak_lg) + assert loaded_charge_container.run_number == charge_container.run_number + assert ( + loaded_charge_container.pixels_id.tolist() + == charge_container.pixels_id.tolist() + ) + assert loaded_charge_container.nevents == charge_container.nevents + assert loaded_charge_container.npixels == charge_container.npixels + assert loaded_charge_container.method == charge_container.method + + def test_access_properties(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) + nevents = 40 + npixels = 10 + charges_hg = np.random.randn(nevents, npixels) + charges_lg = np.random.randn(nevents, npixels) + peak_hg = np.random.randn(nevents, npixels) + peak_lg = np.random.randn(nevents, npixels) + run_number = 1234 + method = "FullWaveformSum" + charge_container = ChargesContainer( + charges_hg=charges_hg, + charges_lg=charges_lg, + peak_hg=peak_hg, + peak_lg=peak_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + method=method, + ) + assert charge_container.run_number == run_number + assert charge_container.pixels_id.tolist() == pixels_id.tolist() + assert charge_container.npixels == npixels + assert charge_container.nevents == nevents + assert charge_container.method == method + + +class TestChargesContainers: + run_number = 1234 + nevents = 140 + npixels = 10 + + def test_create_charge_container(self): + chargesContainers = create_fake_chargeContainers() + assert isinstance(chargesContainers, ChargesContainers) + for key in chargesContainers.containers.keys(): + assert isinstance(key, EventType) + + def test_write_charge_containers(self, tmp_path="/tmp"): + charge_containers = create_fake_chargeContainers() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + for key, container in charge_containers.containers.items(): + writer.write(table_name=f"{key.name}", containers=container) + writer.close() + + def test_from_hdf5(self, tmp_path="/tmp"): + charge_containers = create_fake_chargeContainers() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + for key, container in charge_containers.containers.items(): + writer.write(table_name=f"{key.name}", containers=container) + writer.close() + + loaded_charge_container = next(ChargesContainers.from_hdf5(tmp_path)) + + assert isinstance(loaded_charge_container, ChargesContainers) + for key, container in loaded_charge_container.containers.items(): + assert np.allclose( + container.charges_hg, charge_containers.containers[key].charges_hg + ) + assert np.allclose( + container.charges_lg, charge_containers.containers[key].charges_lg + ) + assert np.allclose( + container.peak_hg, charge_containers.containers[key].peak_hg + ) + assert np.allclose( + container.peak_lg, charge_containers.containers[key].peak_lg + ) + assert container.run_number == charge_containers.containers[key].run_number + assert ( + container.pixels_id.tolist() + == charge_containers.containers[key].pixels_id.tolist() + ) + assert container.nevents == charge_containers.containers[key].nevents + assert container.npixels == charge_containers.containers[key].npixels + assert container.method == charge_containers.containers[key].method + + +if __name__ == "__main__": + pass diff --git a/src/nectarchain/data/container/tests/test_core.py b/src/nectarchain/data/container/tests/test_core.py new file mode 100644 index 00000000..10513a04 --- /dev/null +++ b/src/nectarchain/data/container/tests/test_core.py @@ -0,0 +1,175 @@ +import numpy as np +import pytest +from ctapipe.containers import Container, EventType, Field +from ctapipe.io import HDF5TableWriter + +from nectarchain.data.container import ( + ArrayDataContainer, + NectarCAMContainer, + TriggerMapContainer, +) +from nectarchain.data.container.core import get_array_keys, merge_map_ArrayDataContainer + + +class Container_test(NectarCAMContainer): + field1 = Field(type=np.ndarray, dtype=np.uint16, ndim=1, description="filed1") + field2 = Field(type=np.uint16, description="field2") + field3 = Field(type=np.ndarray, dtype=np.uint8, ndim=2, description="field3") + + +def create_fake_NectarCAMContainer(): + container = Container_test( + field1=np.array([1, 2, 3], dtype=np.uint16), + field2=np.uint16(5), + field3=np.array([[4, 5, 6], [3, 4, 7]], dtype=np.uint8), + ) + container.validate() + return container + + +class TestGetArrayKeys: + def test_get_array_keys(self): + container = create_fake_NectarCAMContainer() + keys = get_array_keys(container) + assert ["field1", "field3"] == keys + + +class TestNectarCAMContainer: + def test_create_NectarCAMContainer(self): + test = create_fake_NectarCAMContainer() + assert isinstance(test, NectarCAMContainer) + + +def create_fake_ArrayDataContainer(): + nevents = TestArrayDataContainer.nevents + npixels = TestArrayDataContainer.npixels + rng = np.random.default_rng() + test = ArrayDataContainer( + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16), + nevents=np.uint64(nevents), + npixels=np.uint16(npixels), + run_number=np.uint16(TestArrayDataContainer.run_number), + camera="TEST", + broken_pixels_hg=rng.integers( + low=0, high=1.1, size=(nevents, npixels), dtype=bool + ), + broken_pixels_lg=rng.integers( + low=0, high=1.1, size=(nevents, npixels), dtype=bool + ), + ucts_timestamp=rng.integers(low=0, high=100, size=(nevents), dtype=np.uint64), + ucts_busy_counter=rng.integers( + low=0, high=100, size=(nevents), dtype=np.uint32 + ), + ucts_event_counter=rng.integers( + low=0, high=100, size=(nevents), dtype=np.uint32 + ), + event_type=rng.integers(low=0, high=1, size=(nevents), dtype=np.uint8), + event_id=rng.integers(low=0, high=1000, size=(nevents), dtype=np.uint32), + trig_pattern_all=rng.integers( + low=0, high=1, size=(nevents, npixels, 4), dtype=bool + ), + trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels), dtype=bool), + multiplicity=rng.integers(low=0, high=1, size=(nevents), dtype=np.uint16), + ) + test.validate() + return test + + +class TestArrayDataContainer: + # THESE TESTs ARE ALSO COVERING THE NectarCAMContainer class# + + nevents = 10 + npixels = 40 + run_number = 1234 + + def test_create_ArrayDataContainer(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16) + nevents = np.uint64(TestArrayDataContainer.nevents) + npixels = np.uint16(TestArrayDataContainer.npixels) + run_number = np.uint16(TestArrayDataContainer.run_number) + arrayDataContainer = ArrayDataContainer( + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + ) + arrayDataContainer.validate() + assert arrayDataContainer.run_number == run_number + assert arrayDataContainer.pixels_id.tolist() == pixels_id.tolist() + assert arrayDataContainer.nevents == nevents + assert arrayDataContainer.npixels == npixels + + def test_write_ArrayDataContainer(self, tmp_path="/tmp"): + arrayDataContainer = create_fake_ArrayDataContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="ArrayDataContainer", containers=arrayDataContainer) + writer.close() + + # Tests that a ChargeContainer object can be loaded from a file and the object is + # correctly initialized. + def test_from_hdf5(self, tmp_path="/tmp"): + arrayDataContainer = create_fake_ArrayDataContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="ArrayDataContainer", containers=arrayDataContainer) + writer.close() + loaded_arrayDataContainer = next(arrayDataContainer.from_hdf5(tmp_path)) + assert isinstance(loaded_arrayDataContainer, ArrayDataContainer) + assert loaded_arrayDataContainer.run_number == arrayDataContainer.run_number + assert ( + loaded_arrayDataContainer.pixels_id.tolist() + == arrayDataContainer.pixels_id.tolist() + ) + assert loaded_arrayDataContainer.nevents == arrayDataContainer.nevents + assert loaded_arrayDataContainer.npixels == arrayDataContainer.npixels + + def test_access_properties(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) + nevents = 40 + npixels = 10 + run_number = 1234 + arrayDataContainer = ArrayDataContainer( + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + ) + assert arrayDataContainer.run_number == run_number + assert arrayDataContainer.pixels_id.tolist() == pixels_id.tolist() + assert arrayDataContainer.npixels == npixels + assert arrayDataContainer.nevents == nevents + + +class TestTriggerMapContainer: + def test_create_TriggerMapContainer(self): + triggerMapContainer = TriggerMapContainer() + triggerMapContainer.containers[ + EventType.FLATFIELD + ] = create_fake_NectarCAMContainer() + triggerMapContainer.containers[ + EventType.MUON + ] = create_fake_NectarCAMContainer() + assert isinstance(triggerMapContainer, TriggerMapContainer) + + def test_merge_map_ArrayDataContainer(self): + triggerMapContainer = TriggerMapContainer() + arrayDataContainer1 = create_fake_ArrayDataContainer() + arrayDataContainer2 = create_fake_ArrayDataContainer() + + triggerMapContainer.containers["1"] = arrayDataContainer1 + triggerMapContainer.containers["2"] = arrayDataContainer2 + + merged = merge_map_ArrayDataContainer(triggerMapContainer) + + assert ( + merged.nevents == arrayDataContainer1.nevents + arrayDataContainer2.nevents + ) diff --git a/src/nectarchain/data/container/tests/test_gain.py b/src/nectarchain/data/container/tests/test_gain.py new file mode 100644 index 00000000..e6a10b62 --- /dev/null +++ b/src/nectarchain/data/container/tests/test_gain.py @@ -0,0 +1,227 @@ +import glob + +import numpy as np +import pytest +from ctapipe.containers import EventType +from ctapipe.io import HDF5TableWriter + +from nectarchain.data.container import GainContainer, SPEfitContainer + + +def create_fake_GainContainer(): + npixels = TestGainContainer.npixels + rng = np.random.default_rng() + gain = GainContainer( + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16), + is_valid=rng.integers(low=0, high=1.1, size=(npixels,), dtype=bool), + high_gain=rng.random(size=(npixels, 3), dtype=np.float64), + low_gain=rng.random(size=(npixels, 3), dtype=np.float64), + ) + gain.validate() + return gain + + +def create_fake_SPEfitContainer(): + npixels = TestSPEfitContainer.npixels + rng = np.random.default_rng() + gain = SPEfitContainer( + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16), + is_valid=rng.integers(low=0, high=1.1, size=(npixels,), dtype=bool), + high_gain=rng.random(size=(npixels, 3), dtype=np.float64), + low_gain=rng.random(size=(npixels, 3), dtype=np.float64), + likelihood=rng.random(size=(npixels,), dtype=np.float64), + p_value=rng.random(size=(npixels,), dtype=np.float64), + pedestal=rng.random(size=(npixels, 3), dtype=np.float64), + pedestalWidth=rng.random(size=(npixels, 3), dtype=np.float64), + resolution=rng.random(size=(npixels, 3), dtype=np.float64), + luminosity=rng.random(size=(npixels, 3), dtype=np.float64), + mean=rng.random(size=(npixels, 3), dtype=np.float64), + n=rng.random(size=(npixels, 3), dtype=np.float64), + pp=rng.random(size=(npixels, 3), dtype=np.float64), + ) + gain.validate() + return gain + + +class TestGainContainer: + npixels = 10 + + def test_create_gain_container(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16) + npixels = np.uint16(TestGainContainer.npixels) + high_gain = np.float64(np.random.randn(npixels, 3)) + low_gain = np.float64(np.random.randn(npixels, 3)) + is_valid = np.array(np.random.randn(npixels), dtype=bool) + gain_container = GainContainer( + pixels_id=pixels_id, + is_valid=is_valid, + high_gain=high_gain, + low_gain=low_gain, + ) + gain_container.validate() + assert gain_container.pixels_id.tolist() == pixels_id.tolist() + assert gain_container.is_valid.tolist() == is_valid.tolist() + assert gain_container.high_gain.tolist() == high_gain.tolist() + assert gain_container.low_gain.tolist() == low_gain.tolist() + + def test_write_gain_container(self, tmp_path="/tmp"): + gain_container = create_fake_GainContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="GainContainer", containers=gain_container) + writer.close() + + def test_from_hdf5(self, tmp_path="/tmp"): + gain_container = create_fake_GainContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="GainContainer", containers=gain_container) + writer.close() + + loaded_gain_container = next(GainContainer.from_hdf5(tmp_path)) + + assert isinstance(loaded_gain_container, GainContainer) + assert ( + gain_container.pixels_id.tolist() + == loaded_gain_container.pixels_id.tolist() + ) + assert ( + gain_container.is_valid.tolist() == loaded_gain_container.is_valid.tolist() + ) + assert ( + gain_container.high_gain.tolist() + == loaded_gain_container.high_gain.tolist() + ) + assert ( + gain_container.low_gain.tolist() == loaded_gain_container.low_gain.tolist() + ) + + +class TestSPEfitContainer: + npixels = 10 + + def test_create_SPEfit_container(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16) + npixels = np.uint16(TestSPEfitContainer.npixels) + high_gain = np.float64(np.random.randn(npixels, 3)) + low_gain = np.float64(np.random.randn(npixels, 3)) + is_valid = np.array(np.random.randn(npixels), dtype=bool) + likelihood = np.float64(np.random.randn(npixels)) + p_value = np.float64(np.random.randn(npixels)) + pedestal = np.float64(np.random.randn(npixels, 3)) + pedestalWidth = np.float64(np.random.randn(npixels, 3)) + resolution = np.float64(np.random.randn(npixels, 3)) + luminosity = np.float64(np.random.randn(npixels, 3)) + mean = np.float64(np.random.randn(npixels, 3)) + n = np.float64(np.random.randn(npixels, 3)) + pp = np.float64(np.random.randn(npixels, 3)) + SPEfit_container = SPEfitContainer( + pixels_id=pixels_id, + is_valid=is_valid, + high_gain=high_gain, + low_gain=low_gain, + likelihood=likelihood, + p_value=p_value, + pedestal=pedestal, + pedestalWidth=pedestalWidth, + resolution=resolution, + luminosity=luminosity, + mean=mean, + n=n, + pp=pp, + ) + SPEfit_container.validate() + assert SPEfit_container.pixels_id.tolist() == pixels_id.tolist() + assert SPEfit_container.is_valid.tolist() == is_valid.tolist() + assert SPEfit_container.high_gain.tolist() == high_gain.tolist() + assert SPEfit_container.low_gain.tolist() == low_gain.tolist() + assert SPEfit_container.likelihood.tolist() == likelihood.tolist() + assert SPEfit_container.p_value.tolist() == p_value.tolist() + assert SPEfit_container.pedestal.tolist() == pedestal.tolist() + assert SPEfit_container.pedestalWidth.tolist() == pedestalWidth.tolist() + assert SPEfit_container.resolution.tolist() == resolution.tolist() + assert SPEfit_container.luminosity.tolist() == luminosity.tolist() + assert SPEfit_container.mean.tolist() == mean.tolist() + assert SPEfit_container.n.tolist() == n.tolist() + assert SPEfit_container.pp.tolist() == pp.tolist() + + def test_write_SPEfit_container(self, tmp_path="/tmp"): + SPEfit_container = create_fake_SPEfitContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="SPEfitContainer", containers=SPEfit_container) + writer.close() + + def test_from_hdf5(self, tmp_path="/tmp"): + SPEfit_container = create_fake_SPEfitContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="SPEfitContainer", containers=SPEfit_container) + writer.close() + + loaded_SPEfit_container = next(SPEfitContainer.from_hdf5(tmp_path)) + + assert isinstance(loaded_SPEfit_container, SPEfitContainer) + assert ( + SPEfit_container.pixels_id.tolist() + == loaded_SPEfit_container.pixels_id.tolist() + ) + assert ( + SPEfit_container.is_valid.tolist() + == loaded_SPEfit_container.is_valid.tolist() + ) + assert ( + SPEfit_container.high_gain.tolist() + == loaded_SPEfit_container.high_gain.tolist() + ) + assert ( + SPEfit_container.low_gain.tolist() + == loaded_SPEfit_container.low_gain.tolist() + ) + assert ( + SPEfit_container.likelihood.tolist() + == loaded_SPEfit_container.likelihood.tolist() + ) + assert ( + SPEfit_container.p_value.tolist() + == loaded_SPEfit_container.p_value.tolist() + ) + assert ( + SPEfit_container.pedestal.tolist() + == loaded_SPEfit_container.pedestal.tolist() + ) + assert ( + SPEfit_container.pedestalWidth.tolist() + == loaded_SPEfit_container.pedestalWidth.tolist() + ) + assert ( + SPEfit_container.resolution.tolist() + == loaded_SPEfit_container.resolution.tolist() + ) + assert ( + SPEfit_container.luminosity.tolist() + == loaded_SPEfit_container.luminosity.tolist() + ) + assert SPEfit_container.mean.tolist() == loaded_SPEfit_container.mean.tolist() + assert SPEfit_container.n.tolist() == loaded_SPEfit_container.n.tolist() + assert SPEfit_container.pp.tolist() == loaded_SPEfit_container.pp.tolist() + + +if __name__ == "__main__": + pass diff --git a/src/nectarchain/data/container/tests/test_waveform.py b/src/nectarchain/data/container/tests/test_waveform.py new file mode 100644 index 00000000..54851e36 --- /dev/null +++ b/src/nectarchain/data/container/tests/test_waveform.py @@ -0,0 +1,215 @@ +import glob + +import numpy as np +import pytest +from ctapipe.containers import EventType +from ctapipe.io import HDF5TableWriter + +from nectarchain.data.container import WaveformsContainer, WaveformsContainers + + +def create_fake_waveformContainer(): + nevents = TestWaveformsContainer.nevents + npixels = TestWaveformsContainer.npixels + nsamples = TestWaveformsContainer.nsamples + + rng = np.random.default_rng() + waveform = WaveformsContainer( + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16), + nevents=np.uint64(nevents), + npixels=np.uint16(npixels), + nsamples=np.uint8(nsamples), + wfs_hg=rng.integers( + low=0, high=1000, size=(nevents, npixels, nsamples), dtype=np.uint16 + ), + wfs_lg=rng.integers( + low=0, high=1000, size=(nevents, npixels, nsamples), dtype=np.uint16 + ), + run_number=np.uint16(TestWaveformsContainer.run_number), + camera="TEST", + broken_pixels_hg=rng.integers( + low=0, high=1.1, size=(nevents, npixels), dtype=bool + ), + broken_pixels_lg=rng.integers( + low=0, high=1.1, size=(nevents, npixels), dtype=bool + ), + ucts_timestamp=rng.integers(low=0, high=100, size=(nevents), dtype=np.uint64), + ucts_busy_counter=rng.integers( + low=0, high=100, size=(nevents), dtype=np.uint32 + ), + ucts_event_counter=rng.integers( + low=0, high=100, size=(nevents), dtype=np.uint32 + ), + event_type=rng.integers(low=0, high=1, size=(nevents), dtype=np.uint8), + event_id=rng.integers(low=0, high=1000, size=(nevents), dtype=np.uint32), + trig_pattern_all=rng.integers( + low=0, high=1, size=(nevents, npixels, 4), dtype=bool + ), + trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels), dtype=bool), + multiplicity=rng.integers(low=0, high=1, size=(nevents), dtype=np.uint16), + ) + waveform.validate() + return waveform + + +def create_fake_waveformContainers(): + waveform_1 = create_fake_waveformContainer() + waveform_2 = create_fake_waveformContainer() + waveform = WaveformsContainers() + waveform.containers[EventType.FLATFIELD] = waveform_1 + waveform.containers[EventType.SKY_PEDESTAL] = waveform_2 + return waveform + + +class TestWaveformsContainer: + run_number = 1234 + nevents = 140 + npixels = 10 + nsamples = 6 + + # Tests that a ChargeContainer object can be created with valid input parameters. + def test_create_waveform_container(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16) + nevents = np.uint64(TestWaveformsContainer.nevents) + npixels = np.uint16(TestWaveformsContainer.npixels) + nsamples = np.uint8(TestWaveformsContainer.nsamples) + + run_number = np.uint16(TestWaveformsContainer.run_number) + wfs_hg = np.uint16(np.random.randn(nevents, npixels, nsamples)) + wfs_lg = np.uint16(np.random.randn(nevents, npixels, nsamples)) + waveform_container = WaveformsContainer( + wfs_hg=wfs_hg, + wfs_lg=wfs_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + nsamples=nsamples, + ) + waveform_container.validate() + assert waveform_container.run_number == run_number + assert waveform_container.pixels_id.tolist() == pixels_id.tolist() + assert waveform_container.nevents == nevents + assert waveform_container.npixels == npixels + assert waveform_container.nsamples == nsamples + + def test_write_waveform_container(self, tmp_path="/tmp"): + waveform_container = create_fake_waveformContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="WaveformsContainer", containers=waveform_container) + writer.close() + + def test_from_hdf5(self, tmp_path="/tmp"): + waveform_container = create_fake_waveformContainer() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + writer.write(table_name="WaveformsContainer", containers=waveform_container) + writer.close() + + loaded_waveform_container = next(WaveformsContainer.from_hdf5(tmp_path)) + + assert isinstance(loaded_waveform_container, WaveformsContainer) + assert np.allclose(loaded_waveform_container.wfs_hg, waveform_container.wfs_hg) + assert np.allclose(loaded_waveform_container.wfs_lg, waveform_container.wfs_lg) + assert loaded_waveform_container.run_number == waveform_container.run_number + assert ( + loaded_waveform_container.pixels_id.tolist() + == waveform_container.pixels_id.tolist() + ) + assert loaded_waveform_container.nevents == waveform_container.nevents + assert loaded_waveform_container.npixels == waveform_container.npixels + assert loaded_waveform_container.nsamples == waveform_container.nsamples + + def test_access_properties(self): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) + nevents = 40 + npixels = 10 + nsamples = 6 + wfs_hg = np.random.randn(nevents, npixels, nsamples) + wfs_lg = np.random.randn(nevents, npixels, nsamples) + run_number = 1234 + waveform_container = WaveformsContainer( + wfs_hg=wfs_hg, + wfs_lg=wfs_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + nsamples=nsamples, + ) + assert waveform_container.run_number == run_number + assert waveform_container.pixels_id.tolist() == pixels_id.tolist() + assert waveform_container.npixels == npixels + assert waveform_container.nevents == nevents + assert waveform_container.nsamples == nsamples + + +class TestWaveformsContainers: + run_number = 1234 + nevents = 140 + npixels = 10 + nsamples = 6 + + def test_create_waveform_container(self): + waveformsContainers = create_fake_waveformContainers() + assert isinstance(waveformsContainers, WaveformsContainers) + for key in waveformsContainers.containers.keys(): + assert isinstance(key, EventType) + + def test_write_waveform_containers(self, tmp_path="/tmp"): + waveform_containers = create_fake_waveformContainers() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + for key, container in waveform_containers.containers.items(): + writer.write(table_name=f"{key.name}", containers=container) + writer.close() + + def test_from_hdf5(self, tmp_path="/tmp"): + waveform_containers = create_fake_waveformContainers() + tmp_path += f"/{np.random.randn(1)[0]}.h5" + writer = HDF5TableWriter( + filename=tmp_path, + mode="w", + group_name="data", + ) + for key, container in waveform_containers.containers.items(): + writer.write(table_name=f"{key.name}", containers=container) + writer.close() + + loaded_waveform_container = next(WaveformsContainers.from_hdf5(tmp_path)) + + assert isinstance(loaded_waveform_container, WaveformsContainers) + for key, container in loaded_waveform_container.containers.items(): + assert np.allclose( + container.wfs_hg, waveform_containers.containers[key].wfs_hg + ) + assert np.allclose( + container.wfs_lg, waveform_containers.containers[key].wfs_lg + ) + assert ( + container.run_number == waveform_containers.containers[key].run_number + ) + assert ( + container.pixels_id.tolist() + == waveform_containers.containers[key].pixels_id.tolist() + ) + assert container.nevents == waveform_containers.containers[key].nevents + assert container.npixels == waveform_containers.containers[key].npixels + assert container.nsamples == waveform_containers.containers[key].nsamples + + +if __name__ == "__main__": + pass diff --git a/src/nectarchain/data/container/tests/test_waveforms.py b/src/nectarchain/data/container/tests/test_waveforms.py deleted file mode 100644 index 33c0d126..00000000 --- a/src/nectarchain/data/container/tests/test_waveforms.py +++ /dev/null @@ -1,110 +0,0 @@ -# import glob - -# import numpy as np -import pytest - -# from ctapipe.instrument import SubarrayDescription - -# from nectarchain.data.container import WaveformsContainer, WaveformsContainerIO -# from nectarchain.makers import WaveformsMaker - -pytest.skip( - "Some classes to be imported here were dropped from nectarchain," - "skipping all these tests entirely", - allow_module_level=True, -) - - -# def create_fake_waveformsContainer(): -# nevents = TestWaveformsContainer.nevents -# npixels = TestWaveformsContainer.npixels -# nsamples = TestWaveformsContainer.nsamples -# rng = np.random.default_rng() -# faked_subarray = SubarrayDescription(name="TEST") -# -# return WaveformsContainer( -# pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]), -# nevents=nevents, -# npixels=npixels, -# wfs_hg=rng.integers(low=0, high=1000, size=(nevents, npixels, nsamples)), -# wfs_lg=rng.integers(low=0, high=1000, size=(nevents, npixels, nsamples)), -# run_number=TestWaveformsContainer.run_number, -# camera="TEST", -# subarray=faked_subarray, -# broken_pixels_hg=rng.integers(low=0, high=1, size=(nevents, npixels)), -# broken_pixels_lg=rng.integers(low=0, high=1, size=(nevents, npixels)), -# ucts_timestamp=rng.integers(low=0, high=100, size=(nevents)), -# ucts_busy_counter=rng.integers(low=0, high=100, size=(nevents)), -# ucts_event_counter=rng.integers(low=0, high=100, size=(nevents)), -# event_type=rng.integers(low=0, high=1, size=(nevents)), -# event_id=rng.integers(low=0, high=1000, size=(nevents)), -# trig_pattern_all=rng.integers(low=0, high=1, size=(nevents, npixels, 4)), -# trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels)), -# multiplicity=rng.integers(low=0, high=1, size=(nevents)), -# ) -# -# -# class TestWaveformsContainer: -# run_number = 1234 -# nevents = 140 -# npixels = 10 -# nsamples = 5 -# -# # Tests that a ChargeContainer object can be created with valid input parameters. -# def test_create_waveform_container(self): -# waveform_container = create_fake_waveformsContainer() -# assert isinstance(waveform_container, WaveformsContainer) -# -# # Tests that the ChargeContainer object can be written to a file and the file is -# # created. -# def test_write_waveform_container(self, tmp_path="/tmp"): -# waveform_container = create_fake_waveformsContainer() -# tmp_path += f"/{np.random.randn(1)[0]}" -# -# WaveformsContainerIO.write(tmp_path, waveform_container) -# -# assert ( -# len(glob.glob(f"{tmp_path}/*_run{TestWaveformsContainer.run_number}.fits")) -# == 1 -# ) -# -# # Tests that a ChargeContainer object can be loaded from a file and the object is -# # correctly initialized. -# def test_load_waveform_container(self, tmp_path="/tmp"): -# waveform_container = create_fake_waveformsContainer() -# tmp_path += f"/{np.random.randn(1)[0]}" -# -# WaveformsContainerIO.write(tmp_path, waveform_container) -# -# loaded_waveform_container = WaveformsContainerIO.load( -# tmp_path, TestWaveformsContainer.run_number -# ) -# -# assert isinstance(loaded_waveform_container, WaveformsContainer) -# assert np.allclose(loaded_waveform_container.wfs_hg, -# waveform_container.wfs_hg) -# assert np.allclose(loaded_waveform_container.wfs_lg, -# waveform_container.wfs_lg) -# assert loaded_waveform_container.run_number == waveform_container.run_number -# assert ( -# loaded_waveform_container.pixels_id.tolist() -# == waveform_container.pixels_id.tolist() -# ) -# assert loaded_waveform_container.nevents == waveform_container.nevents -# assert loaded_waveform_container.npixels == waveform_container.npixels -# assert loaded_waveform_container.nsamples == waveform_container.nsamples -# -# # Tests that the ChargeContainer object can be sorted by event_id and the object -# # is sorted accordingly. -# def test_sort_waveform_container(self): -# waveform_container = create_fake_waveformsContainer() -# -# sorted_waveform_container = WaveformsMaker.sort(waveform_container) -# -# assert sorted_waveform_container.event_id.tolist() == sorted( -# waveform_container.event_id.tolist() -# ) -# -# -# if __name__ == "__main__": -# TestWaveformsContainer().test_create_waveform_container() diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 45da1296..05eb0fd7 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -35,6 +35,15 @@ class WaveformsContainer(ArrayDataContainer): class WaveformsContainers(TriggerMapContainer): + """ + Class representing a container for waveforms from specific runs. + + This class inherits from the `TriggerMapContainer` class and is used to store trigger or slices of data mappings of `WaveformsContainer` instances. + + Attributes: + containers (Field): A field representing the trigger or slices of data mapping of `WaveformsContainer` instances. + """ + containers = Field( default_factory=partial(Map, WaveformsContainer), description="trigger or slices of data mapping of WaveformContainer", diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 7872132e..a8ebf5d5 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -18,9 +18,10 @@ # The DIRAC magic 2 lines ! try: - import DIRAC + with KeepLoggingUnchanged(): + import DIRAC - DIRAC.initialize() + DIRAC.initialize() except ImportError: log.warning("DIRAC probably not installed") pass @@ -47,7 +48,6 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: ) list_path = [Path(chemin) for chemin in list] if len(list_path) == 0: - ############# e = FileNotFoundError(f"run {run_number} is not present in {basepath}") if search_on_GRID: log.warning(e, exc_info=True) @@ -61,7 +61,6 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: else: log.error(e, exc_info=True) raise e - # the pb is here !!!!!! name = list_path[0].name.split(".") name[2] = "*" diff --git a/src/nectarchain/data/tests/test_management.py b/src/nectarchain/data/tests/test_management.py new file mode 100644 index 00000000..5871ed8e --- /dev/null +++ b/src/nectarchain/data/tests/test_management.py @@ -0,0 +1 @@ +import pytest diff --git a/src/nectarchain/makers/component/chargesComponent.py b/src/nectarchain/makers/component/chargesComponent.py index 9a62b11e..c35dc026 100644 --- a/src/nectarchain/makers/component/chargesComponent.py +++ b/src/nectarchain/makers/component/chargesComponent.py @@ -168,8 +168,8 @@ def __call__( broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( event, self._pixels_id ) - self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) - self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) + self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg) + self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg) imageExtractor = __class__._get_imageExtractor( self.method, self.subarray, **self.extractor_kwargs @@ -180,16 +180,16 @@ def __call__( wfs_hg_tmp, self.TEL_ID, constants.HIGH_GAIN, broken_pixels_hg ) ) - self.__charges_hg[f"{name}"].append(__image[0].tolist()) - self.__peak_hg[f"{name}"].append(__image[1].tolist()) + self.__charges_hg[f"{name}"].append(__image[0]) + self.__peak_hg[f"{name}"].append(__image[1]) __image = CtapipeExtractor.get_image_peak_time( imageExtractor( wfs_lg_tmp, self.TEL_ID, constants.LOW_GAIN, broken_pixels_lg ) ) - self.__charges_lg[f"{name}"].append(__image[0].tolist()) - self.__peak_lg[f"{name}"].append(__image[1].tolist()) + self.__charges_lg[f"{name}"].append(__image[0]) + self.__peak_lg[f"{name}"].append(__image[1]) @staticmethod def _get_extractor_kwargs_from_method_and_kwargs(method: str, kwargs: dict): diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveformsComponent.py index c8dd4a5f..74e13237 100644 --- a/src/nectarchain/makers/component/waveformsComponent.py +++ b/src/nectarchain/makers/component/waveformsComponent.py @@ -142,14 +142,14 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): ) name = __class__._get_name_trigger(event.trigger.event_type) - self.__wfs_hg[f"{name}"].append(wfs_hg_tmp.tolist()) - self.__wfs_lg[f"{name}"].append(wfs_lg_tmp.tolist()) + self.__wfs_hg[f"{name}"].append(wfs_hg_tmp) + self.__wfs_lg[f"{name}"].append(wfs_lg_tmp) broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( event, self._pixels_id ) - self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) - self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) + self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg) + self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg) def finish(self, *args, **kwargs): """Make the output container for the selected trigger types.