Add container/component/tool for pedestal estimation

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ name: CI
 on:
   push:
     branches:
-      - master
+      - main
     tags:
       - '**'
   pull_request:
@@ -75,16 +75,15 @@ jobs:
         run: |
           python --version
           echo "Installing additional pip packages"
-          # we install ctapipe using pip to be able to select any commit, e.g. the current master
+          # we install ctapipe using pip to be able to select any commit, e.g. the current main
           pip install \
-             "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" \
-              pytest-cov
+             "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION"
 
           echo "pip install -e ."
-          pip install -e .
+          pip install -e .[test]
 
       - name: Tests
         run: |
-          pytest --cov=nectarchain --cov-report=xml
+          pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml
 
       - uses: codecov/codecov-action@v3

.github/workflows/deploy-ghcr.yml

@@ -4,7 +4,7 @@ on:
   pull_request: []
   push:
     branches:
-      - master
+      - main
   release:
     types: [ published ]
 
@@ -67,7 +67,7 @@ jobs:
       - name: Deploy ${{ env.container }} container ${{ matrix.deffiles[1] }}
         # Otherwise, the container tag is "latest" by default.
         # Don't push the container on a pull request.
-        if: github.event_name != 'pull_request'
+        if: ${{ (github.event_name != 'release') && (github.event_name != 'pull_request') }}
         run: |  
           apptainer push ${{ env.container }}.sif oras://${{ env.registry }}/${{ github.repository }}:${{ matrix.deffiles[1] }}
 

.github/workflows/release-drafter.yml

@@ -3,7 +3,7 @@ name: Release Drafter
 on:
   push:
     branches:
-      - master
+      - main
 
 jobs:
   update_release_draft:

.gitignore

@@ -96,4 +96,4 @@ src/nectarchain/user_scripts/**/test
 
 #VScode
 .vscode/
-
+*.code-workspace

README.md

@@ -1,4 +1,4 @@
-# nectarchain [![Build Status](https://github.com/cta-observatory/nectarchain/workflows/CI/badge.svg?branch=master)](https://github.com/cta-observatory/nectarchain/actions?query=workflow%3ACI+branch%3Amaster)
+# nectarchain [![Build Status](https://github.com/cta-observatory/nectarchain/workflows/CI/badge.svg?branch=main)](https://github.com/cta-observatory/nectarchain/actions?query=workflow%3ACI+branch%3Amain)
 
 Repository for the high level analysis of the NectarCAM data.
 The analysis is heavily based on [ctapipe](https://github.com/cta-observatory/ctapipe), adding custom code for NectarCAM calibration.

pyproject.toml

@@ -39,6 +39,7 @@ dynamic = ["version"]
 test = [
     "pytest",
     "pytest-cov",
+    "pytest-xdist",
 ]
 dev = [
     "setuptools_scm",

src/nectarchain/data/container/__init__.py

@@ -9,3 +9,4 @@
 from .eventSource import *
 from .gainContainer import *
 from .waveformsContainer import *
+from .pedestalContainer import *

src/nectarchain/data/container/pedestalContainer.py

@@ -0,0 +1,66 @@
+import logging
+
+logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+log.handlers = logging.getLogger("__main__").handlers
+
+import numpy as np
+from ctapipe.containers import Field
+
+from .core import NectarCAMContainer
+
+
+class NectarCAMPedestalContainer(NectarCAMContainer):
+    """
+    A container that holds estimated pedestals
+
+    Fields:
+        nsamples (int): The number of samples in the waveforms.
+        nevents (np.ndarray): The number of events used to estimate the pedestals for each pixel.
+        pixels_id (np.ndarray): An array of pixel IDs.
+        ucts_timestamp_min (int): The minimum of the input events UCTS timestamps.
+        ucts_timestamp_max (int): The maximum of the input events UCTS timestamps.
+        pedestal_mean_hg (np.ndarray): An array of high gain mean pedestals.
+        pedestal_mean_lg (np.ndarray): An array of low gain mean pedestals.
+        pedestal_std_hg (np.ndarray): An array of standard deviations of high gain pedestals.
+        pedestal_std_lg (np.ndarray): An array of standard deviations of low gain pedestals.
+    """
+
+    nsamples = Field(
+        type=np.uint8,
+        description="number of samples in the waveforms",
+    )
+
+    nevents = Field(
+        type=np.ndarray, dtype=np.float64, ndim=1,
+        description="number of events used to estimate the pedestals for each pixel",
+    )
+
+    pixels_id = Field(type=np.ndarray, dtype=np.uint16, ndim=1, description="pixel ids")
+
+    ucts_timestamp_min = Field(
+        type=np.uint64,
+        description="minimum of the input events UCTS timestamps",
+    )
+
+    ucts_timestamp_max = Field(
+        type=np.uint64,
+        description="maximum of the input events UCTS timestamps",
+    )
+
+    pedestal_mean_hg = Field(
+        type=np.ndarray, dtype=np.float64, ndim=2, description="high gain mean pedestals"
+    )
+
+    pedestal_mean_lg = Field(
+        type=np.ndarray, dtype=np.float64, ndim=2, description="low gain mean pedestals"
+    )
+
+    pedestal_std_hg = Field(
+        type=np.ndarray, dtype=np.float64, ndim=2,
+        description="high gain pedestals standard deviations"
+    )
+    pedestal_std_lg = Field(
+        type=np.ndarray, dtype=np.float64, ndim=2,
+        description="low gain pedestals standard deviations"
+    )

src/nectarchain/data/container/tests/test_pedestal.py

@@ -0,0 +1,99 @@
+import numpy as np
+import tempfile
+from ctapipe.io import HDF5TableWriter
+from nectarchain.data.container import NectarCAMPedestalContainer
+
+def generate_mock_pedestal_container():
+    # fixed values
+    npixels = 10
+    nevents_max = 100
+    nevents_min = 10
+    nsamples = np.uint8(60)
+    # random values for other fields
+    nevents = np.float64(np.random.randint(nevents_min, nevents_max, size=(npixels,)))
+    pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10], dtype=np.uint16)
+    ucts_timestamp_min = np.uint64(np.random.randint(1e8))
+    ucts_timestamp_max = np.uint64(np.random.randint(1e8 + 100))
+    pedestal_mean_hg = np.float64(np.random.uniform(240, 260, size=(npixels, nsamples)))
+    pedestal_mean_lg = np.float64(np.random.uniform(240, 260, size=(npixels, nsamples)))
+    pedestal_std_hg = np.float64(np.random.normal(size=(npixels, nsamples)))
+    pedestal_std_lg = np.float64(np.random.normal(size=(npixels, nsamples)))
+
+    # create pedestal container
+    pedestal_container = NectarCAMPedestalContainer(
+        nsamples=nsamples,
+        nevents=nevents,
+        pixels_id=pixels_id,
+        ucts_timestamp_min=ucts_timestamp_min,
+        ucts_timestamp_max=ucts_timestamp_max,
+        pedestal_mean_hg=pedestal_mean_hg,
+        pedestal_mean_lg=pedestal_mean_lg,
+        pedestal_std_hg=pedestal_std_hg,
+        pedestal_std_lg=pedestal_std_lg
+    )
+    pedestal_container.validate()
+
+    # create dictionary that duplicates content
+    dict = {'nsamples': nsamples,
+            'nevents': nevents,
+            'pixels_id': pixels_id,
+            'ucts_timestamp_min': ucts_timestamp_min,
+            'ucts_timestamp_max': ucts_timestamp_max,
+            'pedestal_mean_hg': pedestal_mean_hg,
+            'pedestal_mean_lg': pedestal_mean_lg,
+            'pedestal_std_hg': pedestal_std_hg,
+            'pedestal_std_lg': pedestal_std_lg,
+            }
+
+    # return both container and input content
+    return pedestal_container, dict
+
+class TestNectarCAMPedestalContainer:
+
+    def test_create_pedestal_container_mem(self):
+        # create mock pedestal container
+        pedestal_container, dict = generate_mock_pedestal_container()
+
+        # check that all fields are filled correctly with input values
+        assert pedestal_container.nsamples == dict['nsamples']
+        assert pedestal_container.nevents.tolist() == dict['nevents'].tolist()
+        assert pedestal_container.ucts_timestamp_min == dict['ucts_timestamp_min']
+        assert pedestal_container.ucts_timestamp_max == dict['ucts_timestamp_max']
+        assert np.allclose(pedestal_container.pedestal_mean_hg, dict['pedestal_mean_hg'])
+        assert np.allclose(pedestal_container.pedestal_mean_lg, dict['pedestal_mean_lg'])
+        assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg'])
+        assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg'])
+
+    # FIXME
+    # Guillaume is working on generalizing the fromhdf5 method to all containers
+    # This test should work once it's done
+    # def test_pedestal_container_io(self):
+    #     input_pedestal_container, dict = generate_mock_pedestal_container()
+    #     with tempfile.TemporaryDirectory() as tmpdirname:
+    #         outpath = tmpdirname+"/pedestal_container_0.h5"
+    #         writer = HDF5TableWriter(
+    #             filename=outpath,
+    #             mode="w",
+    #             group_name="data",
+    #         )
+    #         writer.write(table_name="NectarCAMPedestalContainer",
+    #                      containers=input_pedestal_container)
+    #         writer.close()
+    #
+    #         pedestal_container = next(NectarCAMPedestalContainer.from_hdf5(outpath))
+    #
+    #         #check that container is an instance of the proper class
+    #         assert isinstance(pedestal_container,NectarCAMPedestalContainer)
+    #
+    #         #check content
+    #         assert pedestal_container.nsamples == dict['nsamples']
+    #         assert pedestal_container.nevents.tolist() == dict['nevents'].tolist()
+    #         assert pedestal_container.ucts_timestamp_min == dict['ucts_timestamp_min']
+    #         assert pedestal_container.ucts_timestamp_max == dict['ucts_timestamp_max']
+    #         assert np.allclose(pedestal_container.pedestal_mean_hg, dict['pedestal_mean_hg'])
+    #         assert np.allclose(pedestal_container.pedestal_mean_lg, dict['pedestal_mean_lg'])
+    #         assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg'])
+    #         assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg'])
+
+if __name__ == "__main__":
+    pass