adding CI tests for all DQM modules - initial + cleaning code

cta-observatory · Jul 3, 2024 · f14f40e · f14f40e
1 parent 380f51e
commit f14f40e
Show file tree

Hide file tree

Showing 17 changed files with 256 additions and 49 deletions.
diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py
@@ -4,11 +4,11 @@
 import numpy as np
 from astropy import time as astropytime
 from ctapipe.coordinates import EngineeringCameraFrame
-from ctapipe.instrument import CameraGeometry
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class CameraMonitoring(DQMSummary):
     def __init__(self, gaink):
@@ -139,7 +139,6 @@ def GetResults(self):
         return self.CameraMonitoring_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         try:
             fig, disp = plt.subplots()
             disp = CameraDisplay(self.camera)

diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py
@@ -3,10 +3,11 @@
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.image import LocalPeakWindowSum
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 from traitlets.config.loader import Config
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class ChargeIntegrationHighLowGain(DQMSummary):
     def __init__(self, gaink):
@@ -87,8 +88,9 @@ def ProcessEvent(self, evt, noped):
 
         waveform = evt.r0.tel[0].waveform[self.k]
 
+        ped = np.mean(waveform[:, 20])
+
         if noped:
-            ped = np.mean(waveform[:, 20])
             w_noped = waveform - ped
             output = self.integrator(
                 w_noped, 0, np.zeros(self.Pix, dtype=int), self.pixelBAD

diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py
@@ -1,11 +1,12 @@
 import numpy as np
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
 
-class MeanCameraDisplay_HighLowGain(DQMSummary):
+
+class MeanCameraDisplayHighLowGain(DQMSummary):
     def __init__(self, gaink):
         self.k = gaink
         self.Pix = None
@@ -22,7 +23,7 @@ def __init__(self, gaink):
         self.CameraAverage_ped1 = []
         self.CameraAverage_overEvents = None
         self.CameraAverage_overEvents_overSamp = None
-        self.CameraAverage_ped_overEvents  = None
+        self.CameraAverage_ped_overEvents = None
         self.CameraAverage_ped_overEvents_overSamp = None
         self.MeanCameraDisplay_Results_Dict = {}
         self.MeanCameraDisplay_Figures_Dict = {}
@@ -42,7 +43,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1):
 
         self.cmap = "gnuplot2"
 
-
     def ProcessEvent(self, evt, noped):
         self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels
         pixel = evt.nectarcam.tel[0].svc.pixel_ids
@@ -86,7 +86,6 @@ def FinishRun(self):
             )
 
     def GetResults(self):
-
         # ASSIGN RESUTLS TO DICT
         if self.k == 0:
             if self.counter_evt > 0:
@@ -125,7 +124,6 @@ def GetResults(self):
         return self.MeanCameraDisplay_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         # titles = ['All', 'Pedestals']
         if self.k == 0:
             gain_c = "High"

diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py
@@ -1,7 +1,8 @@
 import numpy as np
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class MeanWaveFormsHighLowGain(DQMSummary):
     def __init__(self, gaink):
@@ -70,7 +71,6 @@ def FinishRun(self):
         return None
 
     def GetResults(self):
-
         # ASSIGN RESUTLS TO DICT
         if self.k == 0:
             self.MeanWaveForms_Results_Dict[
@@ -93,7 +93,6 @@ def GetResults(self):
         return self.MeanWaveForms_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         wf_list = np.array(self.wf_list_plot)
 
         counter_fig = 0

diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py
@@ -1,9 +1,10 @@
 import numpy as np
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 __all__ = ["PixelParticipationHighLowGain"]
 
 

diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py
@@ -1,7 +1,8 @@
 import numpy as np
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class PixelTimelineHighLowGain(DQMSummary):
     def __init__(self, gaink):

diff --git a/src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite b/src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite
diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py
@@ -9,7 +9,7 @@
 from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN
 from db_utils import DQMDB
 from matplotlib import pyplot as plt
-from mean_camera_display import MeanCameraDisplay_HighLowGain
+from mean_camera_display import MeanCameraDisplayHighLowGain
 from mean_waveforms import MeanWaveFormsHighLowGain
 from pixel_participation import PixelParticipationHighLowGain
 from pixel_timeline import PixelTimelineHighLowGain
@@ -145,8 +145,8 @@ def CreateFigFolder(name, type):
 a = TriggerStatistics(HIGH_GAIN)
 b = MeanWaveFormsHighLowGain(HIGH_GAIN)
 c = MeanWaveFormsHighLowGain(LOW_GAIN)
-d = MeanCameraDisplay_HighLowGain(HIGH_GAIN)
-e = MeanCameraDisplay_HighLowGain(LOW_GAIN)
+d = MeanCameraDisplayHighLowGain(HIGH_GAIN)
+e = MeanCameraDisplayHighLowGain(LOW_GAIN)
 f = ChargeIntegrationHighLowGain(HIGH_GAIN)
 g = ChargeIntegrationHighLowGain(LOW_GAIN)
 h = CameraMonitoring(HIGH_GAIN)

diff --git a/src/nectarchain/dqm/tests/__init__.py b/src/nectarchain/dqm/tests/__init__.py
diff --git a/src/nectarchain/dqm/tests/test_camera_monitoring.py b/src/nectarchain/dqm/tests/test_camera_monitoring.py
@@ -0,0 +1,37 @@
+from ctapipe.io import EventSource
+from ctapipe_io_nectarcam.constants import HIGH_GAIN
+from traitlets.config import Config
+
+from nectarchain.dqm.camera_monitoring import CameraMonitoring
+
+
+class TestCameraMonitoring:
+    run_number = 3798
+    max_events = 1
+
+    def test_camera_monitoring(self):
+        path1 = "/Users/hashkar/Desktop/ashkar_nectar/"
+        path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz"
+        path = path1 + path2
+
+        config = None
+
+        config = Config(
+            dict(
+                NectarCAMEventSource=dict(
+                    NectarCAMR0Corrections=dict(
+                        calibration_path=None,
+                        apply_flatfield=False,
+                        select_gain=False,
+                    )
+                )
+            )
+        )
+        print(path)
+
+        reader1 = EventSource(input_url=path, config=config, max_events=1)
+
+        Pix, Samp = CameraMonitoring(HIGH_GAIN).DefineForRun(reader1)
+        print(Pix, Samp)
+        # self.assertEqual(Pix + Samp, 1915)
+        assert Pix + Samp == 1915
diff --git a/src/nectarchain/dqm/tests/test_charge_integration.py b/src/nectarchain/dqm/tests/test_charge_integration.py
@@ -0,0 +1,37 @@
+from ctapipe.io import EventSource
+from ctapipe_io_nectarcam.constants import HIGH_GAIN
+from traitlets.config import Config
+
+from nectarchain.dqm.charge_integration import ChargeIntegrationHighLowGain
+
+
+class TestChargeIntegrationHighLowGain:
+    run_number = 3798
+    max_events = 1
+
+    def test_charge_integration(self):
+        path1 = "/Users/hashkar/Desktop/ashkar_nectar/"
+        path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz"
+        path = path1 + path2
+
+        config = None
+
+        config = Config(
+            dict(
+                NectarCAMEventSource=dict(
+                    NectarCAMR0Corrections=dict(
+                        calibration_path=None,
+                        apply_flatfield=False,
+                        select_gain=False,
+                    )
+                )
+            )
+        )
+        print(path)
+
+        reader1 = EventSource(input_url=path, config=config, max_events=1)
+
+        Pix, Samp = ChargeIntegrationHighLowGain(HIGH_GAIN).DefineForRun(reader1)
+        print(Pix, Samp)
+        # self.assertEqual(Pix + Samp, 1915)
+        assert Pix + Samp == 1915
diff --git a/src/nectarchain/dqm/tests/test_mean_camera_display.py b/src/nectarchain/dqm/tests/test_mean_camera_display.py
@@ -0,0 +1,37 @@
+from ctapipe.io import EventSource
+from ctapipe_io_nectarcam.constants import HIGH_GAIN
+from traitlets.config import Config
+
+from nectarchain.dqm.mean_camera_display import MeanCameraDisplayHighLowGain
+
+
+class TestMeanCameraDisplayHighLowGain:
+    run_number = 3798
+    max_events = 1
+
+    def test_mean_camera_display(self):
+        path1 = "/Users/hashkar/Desktop/ashkar_nectar/"
+        path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz"
+        path = path1 + path2
+
+        config = None
+
+        config = Config(
+            dict(
+                NectarCAMEventSource=dict(
+                    NectarCAMR0Corrections=dict(
+                        calibration_path=None,
+                        apply_flatfield=False,
+                        select_gain=False,
+                    )
+                )
+            )
+        )
+        print(path)
+
+        reader1 = EventSource(input_url=path, config=config, max_events=1)
+
+        Pix, Samp = MeanCameraDisplayHighLowGain(HIGH_GAIN).DefineForRun(reader1)
+        print(Pix, Samp)
+        # self.assertEqual(Pix + Samp, 1915)
+        assert Pix + Samp == 1915
diff --git a/src/nectarchain/dqm/tests/test_mean_wavefomrs.py b/src/nectarchain/dqm/tests/test_mean_wavefomrs.py
@@ -0,0 +1,37 @@
+from ctapipe.io import EventSource
+from ctapipe_io_nectarcam.constants import HIGH_GAIN
+from traitlets.config import Config
+
+from nectarchain.dqm.mean_waveforms import MeanWaveFormsHighLowGain
+
+
+class TestMeanWaveForms:
+    run_number = 3798
+    max_events = 1
+
+    def test_mean_waveforms(self):
+        path1 = "/Users/hashkar/Desktop/ashkar_nectar/"
+        path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz"
+        path = path1 + path2
+
+        config = None
+
+        config = Config(
+            dict(
+                NectarCAMEventSource=dict(
+                    NectarCAMR0Corrections=dict(
+                        calibration_path=None,
+                        apply_flatfield=False,
+                        select_gain=False,
+                    )
+                )
+            )
+        )
+        print(path)
+
+        reader1 = EventSource(input_url=path, config=config, max_events=1)
+
+        Pix, Samp = MeanWaveFormsHighLowGain(HIGH_GAIN).DefineForRun(reader1)
+        print(Pix, Samp)
+        # self.assertEqual(Pix + Samp, 1915)
+        assert Pix + Samp == 1915
diff --git a/src/nectarchain/dqm/tests/test_pixel_participation.py b/src/nectarchain/dqm/tests/test_pixel_participation.py
@@ -1,30 +1,21 @@
-import os
-import sys
-import unittest
-
-import numpy as np
 from ctapipe.io import EventSource
+from ctapipe_io_nectarcam.constants import HIGH_GAIN
 from traitlets.config import Config
 
 from nectarchain.dqm.pixel_participation import PixelParticipationHighLowGain
 
-script_dir = os.path.dirname(os.path.abspath(__file__))
-dqm_dir = os.path.abspath(os.path.join(script_dir, "../../dqm"))
-# Add the directory to sys.path
-sys.path.insert(0, dqm_dir)
-
 
-class TestPixelParticipation(unittest.TestCase):
-    run_number = 3938
+class TestPixelParticipation:
+    run_number = 3798
     max_events = 1
 
     def test_pixel_participation(self):
-        # NectarPath = "/Users/hashkar/Desktop/ashkar_nectar/data/"
-        # path1 = "NectarCAM.Run3731.0000.fits.fz"
-        # path = f"{NectarPath}/runs/{path1}"
-        path1 = "$NECTARDATA/runs/NectarCAM.Run3794.0000.fits.fz"
+        path1 = "/Users/hashkar/Desktop/ashkar_nectar/"
+        path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz"
+        path = path1 + path2
 
         config = None
+
         config = Config(
             dict(
                 NectarCAMEventSource=dict(
@@ -36,12 +27,11 @@ def test_pixel_participation(self):
                 )
             )
         )
-        reader1 = EventSource(
-            input_url=path1, config=config, max_events=TestPixelParticipation.max_events
-        )
-        Pix, Samp = PixelParticipationHighLowGain.DefineForRun(reader1)
-        self.assertEqual(np.add(Pix, Samp), 1915)
+        print(path)
 
+        reader1 = EventSource(input_url=path, config=config, max_events=1)
 
-if __name__ == "__main__":
-    unittest.main()
+        Pix, Samp = PixelParticipationHighLowGain(HIGH_GAIN).DefineForRun(reader1)
+        print(Pix, Samp)
+        # self.assertEqual(Pix + Samp, 1915)
+        assert Pix + Samp == 1915