New implementation of the nectarchain module

.gitignore

@@ -64,7 +64,7 @@ distribute-*.tar.gz
 .ropeproject
 
 # ipython/jupyter notebooks and checkpoints
-*.ipynb
+#*.ipynb
 .ipynb_checkpoints
 .cache*
 .cache
@@ -92,6 +92,7 @@ src/nectarchain/user_scripts/**/test
 **.log
 **.pdf
 **.csv
+**.png
 
 #VScode
 .vscode/

src/nectarchain/data/container/__init__.py

@@ -0,0 +1,5 @@
+from .chargesContainer import *
+from .core import NectarCAMContainer,ArrayDataContainer,TriggerMapContainer,merge_map_ArrayDataContainer,get_array_keys
+from .waveformsContainer import *
+from .gainContainer import *
+from .eventSource import *

src/nectarchain/data/container/chargesContainer.py

@@ -0,0 +1,40 @@
+import logging
+
+logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+log.handlers = logging.getLogger("__main__").handlers
+
+import os
+from abc import ABC
+from pathlib import Path
+
+import numpy as np
+from astropy.io import fits
+from ctapipe.containers import Field,partial,Map
+
+from .core import ArrayDataContainer,TriggerMapContainer
+
+__all__ = ["ChargesContainer","ChargesContainers"]
+class ChargesContainer(ArrayDataContainer):
+    """
+    A container that holds information about charges from a specific run.
+
+    Fields:
+      charges_hg (np.ndarray): An array of high gain charges.
+      charges_lg (np.ndarray): An array of low gain charges.
+      peak_hg (np.ndarray): An array of high gain peak time.
+      peak_lg (np.ndarray): An array of low gain peak time.
+      method (str): The charge extraction method used.
+    """
+
+    charges_hg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The high gain charges")
+    charges_lg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The low gain charges")
+    peak_hg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The high gain peak time")
+    peak_lg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The low gain peak time")
+    method = Field(type=str, description="The charge extraction method used")
+
+
+class ChargesContainers(TriggerMapContainer):
+    containers = Field(default_factory=partial(Map, ChargesContainer),
+                       description = "trigger or slices of data mapping of ChargesContainer"
+                       )

src/nectarchain/data/container/core.py

@@ -1,19 +1,43 @@
 import logging
 
-import numpy as np
-from ctapipe.containers import Container, Field, Map, partial
-
 logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
 log.handlers = logging.getLogger("__main__").handlers
 
+import importlib
+import copy
+import numpy as np
+from pathlib import Path
+from ctapipe.containers import Field,Container,partial,Map
+from ctapipe.core.container import FieldValidationError
+from ctapipe.containers import EventType
+from ctapipe.io import HDF5TableReader
 
-__all__ = ["ArrayDataContainer", "TriggerMapContainer"]
+from tables.exceptions import NoSuchNodeError
 
 
+__all__ = ["ArrayDataContainer","TriggerMapContainer","get_array_keys","merge_map_ArrayDataContainer"]
+def get_array_keys(container : Container) : 
+        keys = []
+        for field in container.fields : 
+            if field.type == np.ndarray : 
+                keys.append(field.key)
+        return keys
 class NectarCAMContainer(Container):
     """base class for the NectarCAM containers. This contaner cannot berecursive,
     to be directly written with a HDF5TableWriter"""
+
+    @staticmethod
+    def _container_from_hdf5(path,container_class) : 
+        if isinstance(path,str) : 
+            path = Path(path)
+
+        container = container_class()
+        with HDF5TableReader(path) as reader :
+            tableReader = reader.read(table_name = f"/data/{container_class.__name__}", containers = container_class)
+            container = next(tableReader)
+
+        yield container
 
 
 class ArrayDataContainer(NectarCAMContainer):
@@ -39,15 +63,15 @@
     """
 
     run_number = Field(
-        type=int,
+        type=np.uint16,
         description="run number associated to the waveforms",
     )
     nevents = Field(
-        type=int,
+        type=np.uint64,
         description="number of events",
     )
     npixels = Field(
-        type=int,
+        type=np.uint16,
         description="number of effective pixels",
     )
     pixels_id = Field(type=np.ndarray, dtype=np.uint16, ndim=1, description="pixel ids")
@@ -85,8 +109,103 @@
     )
 
 
-class TriggerMapContainer(Container):
-    containers = Field(
-        default_factory=partial(Map, Container),
-        description="trigger mapping of Container",
-    )
+    @staticmethod
+    def _container_from_hdf5(path,container_class,slice_index = None) : 
+        if isinstance(path,str) : 
+            path = Path(path)
+        module = importlib.import_module(f'{container_class.__module__}')
+        container = eval(f"module.{container_class.__name__}s")()
+
+
+        with HDF5TableReader(path) as reader : 
+            if len(reader._h5file.root.__members__) > 1 and slice_index is None:
+                log.info(f"reading {container_class.__name__}s containing {len(reader._h5file.root.__members__)} slices, will return a generator")
+                for data in reader._h5file.root.__members__ : 
+                    #container.containers[data] = eval(f"module.{container_class.__name__}s")()
+                    for key,trigger in EventType.__members__.items() : 
+                        try : 
+                            waveforms_data = eval(f"reader._h5file.root.{data}.__members__") 
+                            _mask = [container_class.__name__ in _word for _word in waveforms_data] 
+                            _waveforms_data = np.array(waveforms_data)[_mask]
+                            if len(_waveforms_data) == 1 : 
+                                tableReader = reader.read(table_name = f"/{data}/{_waveforms_data[0]}/{trigger.name}", containers = container_class)
+                                #container.containers[data].containers[trigger] = next(tableReader)
+                                container.containers[trigger] = next(tableReader)
+
+                            else : 
+                                log.info(f"there is {len(_waveforms_data)} entry corresponding to a {container_class} table save, unable to load")
+                        except NoSuchNodeError as err:
+                            log.warning(err)
+                        except Exception as err:
+                            log.error(err,exc_info = True)
+                            raise err
+                    yield container
+            else : 
+                if slice_index is None : 
+                    log.info(f"reading {container_class.__name__}s containing a single slice, will return the {container_class.__name__}s instance")
+                    data = "data"
+                else :
+                    log.info(f"reading slice {slice_index} of {container_class.__name__}s, will return the {container_class.__name__}s instance")
+                    data = f"data_{slice_index}"
+                for key,trigger in EventType.__members__.items() : 
+                    try : 
+                        container_data = eval(f"reader._h5file.root.{data}.__members__") 
+                        _mask =[container_class.__name__ in _word for _word in container_data]
+                        _container_data = np.array(container_data)[_mask]
+                        if len(_container_data) == 1 : 
+                            tableReader = reader.read(table_name = f"/{data}/{_container_data[0]}/{trigger.name}", containers = container_class)
+                            container.containers[trigger] = next(tableReader)
+                        else : 
+                            log.info(f"there is {len(_container_data)} entry corresponding to a {container_class} table save, unable to load")
+                    except NoSuchNodeError as err:
+                        log.warning(err)
+                    except Exception as err:
+                        log.error(err,exc_info = True)
+                        raise err
+                yield container
+        return container 
+
+    @classmethod
+    def from_hdf5(cls,path,slice_index = None) : 
+        return cls._container_from_hdf5(path,slice_index=slice_index,container_class=cls)
+
+
+
+
+
+class TriggerMapContainer(Container) : 
+    containers = Field(default_factory=partial(Map, Container),
+                       description = "trigger mapping of Container"
+                       )
+
+    def is_empty(self) : 
+        return len(self.containers.keys())==0
+
+    def validate(self) : 
+        super().validate()
+        for i,container in enumerate(self.containers) : 
+            if i==0 :
+                container_type = type(container)
+            else : 
+                if not(isinstance(container,container_type)) : 
+                    raise FieldValidationError("all the containers mapped must have the same type to be merged ")
+
+def merge_map_ArrayDataContainer(triggerMapContainer : TriggerMapContainer) : 
+    triggerMapContainer.validate()
+    keys = list(triggerMapContainer.containers.keys())
+    output_container = copy.deepcopy(triggerMapContainer.containers[keys[0]])
+    for key in keys[1:] : 
+        for field in get_array_keys(triggerMapContainer.containers[key]):
+            output_container[field] = np.concatenate((output_container[field],triggerMapContainer.containers[key][field]),axis = 0)
+        if "nevents" in output_container.fields : 
+            output_container.nevents += triggerMapContainer.containers[key].nevents
+    return output_container
+
+#class TriggerMapArrayDataContainer(TriggerMapContainer):
+#    containers = Field(default_factory=partial(Map, ArrayDataContainer),
+#                       description = "trigger mapping of arrayDataContainer"
+#                       )
+
+
+
+