From f14f40e9dcd70232aaf8badc08575420e0e9ef73 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Wed, 3 Jul 2024 15:23:34 +0200 Subject: [PATCH] adding CI tests for all DQM modules - initial + cleaning code --- src/nectarchain/dqm/camera_monitoring.py | 5 +-- src/nectarchain/dqm/charge_integration.py | 6 ++- src/nectarchain/dqm/mean_camera_display.py | 10 ++--- src/nectarchain/dqm/mean_waveforms.py | 5 +-- src/nectarchain/dqm/pixel_participation.py | 3 +- src/nectarchain/dqm/pixel_timeline.py | 3 +- .../nectarcam_monitoring_db_2022-02-24.sqlite | 0 src/nectarchain/dqm/start_dqm.py | 6 +-- src/nectarchain/dqm/tests/__init__.py | 5 --- .../dqm/tests/test_camera_monitoring.py | 37 +++++++++++++++++++ .../dqm/tests/test_charge_integration.py | 37 +++++++++++++++++++ .../dqm/tests/test_mean_camera_display.py | 37 +++++++++++++++++++ .../dqm/tests/test_mean_wavefomrs.py | 37 +++++++++++++++++++ .../dqm/tests/test_pixel_participation.py | 36 +++++++----------- .../dqm/tests/test_pixel_timeline.py | 37 +++++++++++++++++++ .../dqm/tests/test_trigger_statistics.py | 37 +++++++++++++++++++ src/nectarchain/dqm/trigger_statistics.py | 4 +- 17 files changed, 256 insertions(+), 49 deletions(-) create mode 100644 src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite delete mode 100644 src/nectarchain/dqm/tests/__init__.py create mode 100644 src/nectarchain/dqm/tests/test_camera_monitoring.py create mode 100644 src/nectarchain/dqm/tests/test_charge_integration.py create mode 100644 src/nectarchain/dqm/tests/test_mean_camera_display.py create mode 100644 src/nectarchain/dqm/tests/test_mean_wavefomrs.py create mode 100644 src/nectarchain/dqm/tests/test_pixel_timeline.py create mode 100644 src/nectarchain/dqm/tests/test_trigger_statistics.py diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index 57af29e9..686fb870 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -4,11 +4,11 @@ import numpy as np from astropy import time as astropytime from ctapipe.coordinates import EngineeringCameraFrame -from ctapipe.instrument import CameraGeometry from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class CameraMonitoring(DQMSummary): def __init__(self, gaink): @@ -139,7 +139,6 @@ def GetResults(self): return self.CameraMonitoring_Results_Dict def PlotResults(self, name, FigPath): - try: fig, disp = plt.subplots() disp = CameraDisplay(self.camera) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 9416dd21..d3fa673b 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -3,10 +3,11 @@ from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.image import LocalPeakWindowSum from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt from traitlets.config.loader import Config +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class ChargeIntegrationHighLowGain(DQMSummary): def __init__(self, gaink): @@ -87,8 +88,9 @@ def ProcessEvent(self, evt, noped): waveform = evt.r0.tel[0].waveform[self.k] + ped = np.mean(waveform[:, 20]) + if noped: - ped = np.mean(waveform[:, 20]) w_noped = waveform - ped output = self.integrator( w_noped, 0, np.zeros(self.Pix, dtype=int), self.pixelBAD diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index 43baf4b7..dcad59b5 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -1,11 +1,12 @@ import numpy as np from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary -class MeanCameraDisplay_HighLowGain(DQMSummary): + +class MeanCameraDisplayHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink self.Pix = None @@ -22,7 +23,7 @@ def __init__(self, gaink): self.CameraAverage_ped1 = [] self.CameraAverage_overEvents = None self.CameraAverage_overEvents_overSamp = None - self.CameraAverage_ped_overEvents = None + self.CameraAverage_ped_overEvents = None self.CameraAverage_ped_overEvents_overSamp = None self.MeanCameraDisplay_Results_Dict = {} self.MeanCameraDisplay_Figures_Dict = {} @@ -42,7 +43,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.cmap = "gnuplot2" - def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -86,7 +86,6 @@ def FinishRun(self): ) def GetResults(self): - # ASSIGN RESUTLS TO DICT if self.k == 0: if self.counter_evt > 0: @@ -125,7 +124,6 @@ def GetResults(self): return self.MeanCameraDisplay_Results_Dict def PlotResults(self, name, FigPath): - # titles = ['All', 'Pedestals'] if self.k == 0: gain_c = "High" diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py index 7e17b2cb..bd02cdb3 100644 --- a/src/nectarchain/dqm/mean_waveforms.py +++ b/src/nectarchain/dqm/mean_waveforms.py @@ -1,7 +1,8 @@ import numpy as np -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class MeanWaveFormsHighLowGain(DQMSummary): def __init__(self, gaink): @@ -70,7 +71,6 @@ def FinishRun(self): return None def GetResults(self): - # ASSIGN RESUTLS TO DICT if self.k == 0: self.MeanWaveForms_Results_Dict[ @@ -93,7 +93,6 @@ def GetResults(self): return self.MeanWaveForms_Results_Dict def PlotResults(self, name, FigPath): - wf_list = np.array(self.wf_list_plot) counter_fig = 0 diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py index dbdd5be7..7eaad8a9 100644 --- a/src/nectarchain/dqm/pixel_participation.py +++ b/src/nectarchain/dqm/pixel_participation.py @@ -1,9 +1,10 @@ import numpy as np from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + __all__ = ["PixelParticipationHighLowGain"] diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py index e120f561..5437ff42 100644 --- a/src/nectarchain/dqm/pixel_timeline.py +++ b/src/nectarchain/dqm/pixel_timeline.py @@ -1,7 +1,8 @@ import numpy as np -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class PixelTimelineHighLowGain(DQMSummary): def __init__(self, gaink): diff --git a/src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite b/src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite new file mode 100644 index 00000000..e69de29b diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index f6c91701..1b70e310 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -9,7 +9,7 @@ from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN from db_utils import DQMDB from matplotlib import pyplot as plt -from mean_camera_display import MeanCameraDisplay_HighLowGain +from mean_camera_display import MeanCameraDisplayHighLowGain from mean_waveforms import MeanWaveFormsHighLowGain from pixel_participation import PixelParticipationHighLowGain from pixel_timeline import PixelTimelineHighLowGain @@ -145,8 +145,8 @@ def CreateFigFolder(name, type): a = TriggerStatistics(HIGH_GAIN) b = MeanWaveFormsHighLowGain(HIGH_GAIN) c = MeanWaveFormsHighLowGain(LOW_GAIN) -d = MeanCameraDisplay_HighLowGain(HIGH_GAIN) -e = MeanCameraDisplay_HighLowGain(LOW_GAIN) +d = MeanCameraDisplayHighLowGain(HIGH_GAIN) +e = MeanCameraDisplayHighLowGain(LOW_GAIN) f = ChargeIntegrationHighLowGain(HIGH_GAIN) g = ChargeIntegrationHighLowGain(LOW_GAIN) h = CameraMonitoring(HIGH_GAIN) diff --git a/src/nectarchain/dqm/tests/__init__.py b/src/nectarchain/dqm/tests/__init__.py deleted file mode 100644 index 74050196..00000000 --- a/src/nectarchain/dqm/tests/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .db_utils import DQMDB -from .dqm_summary_processor import DQMSummary -from .pixel_participation import PixelParticipationHighLowGain - -__all__ = ["DQMDB", "DQMSummary", "PixelParticipationHighLowGain"] diff --git a/src/nectarchain/dqm/tests/test_camera_monitoring.py b/src/nectarchain/dqm/tests/test_camera_monitoring.py new file mode 100644 index 00000000..e673ada9 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_camera_monitoring.py @@ -0,0 +1,37 @@ +from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from traitlets.config import Config + +from nectarchain.dqm.camera_monitoring import CameraMonitoring + + +class TestCameraMonitoring: + run_number = 3798 + max_events = 1 + + def test_camera_monitoring(self): + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = CameraMonitoring(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/tests/test_charge_integration.py b/src/nectarchain/dqm/tests/test_charge_integration.py new file mode 100644 index 00000000..edc03770 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_charge_integration.py @@ -0,0 +1,37 @@ +from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from traitlets.config import Config + +from nectarchain.dqm.charge_integration import ChargeIntegrationHighLowGain + + +class TestChargeIntegrationHighLowGain: + run_number = 3798 + max_events = 1 + + def test_charge_integration(self): + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = ChargeIntegrationHighLowGain(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/tests/test_mean_camera_display.py b/src/nectarchain/dqm/tests/test_mean_camera_display.py new file mode 100644 index 00000000..0c9e8e76 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_mean_camera_display.py @@ -0,0 +1,37 @@ +from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from traitlets.config import Config + +from nectarchain.dqm.mean_camera_display import MeanCameraDisplayHighLowGain + + +class TestMeanCameraDisplayHighLowGain: + run_number = 3798 + max_events = 1 + + def test_mean_camera_display(self): + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = MeanCameraDisplayHighLowGain(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/tests/test_mean_wavefomrs.py b/src/nectarchain/dqm/tests/test_mean_wavefomrs.py new file mode 100644 index 00000000..9c33caee --- /dev/null +++ b/src/nectarchain/dqm/tests/test_mean_wavefomrs.py @@ -0,0 +1,37 @@ +from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from traitlets.config import Config + +from nectarchain.dqm.mean_waveforms import MeanWaveFormsHighLowGain + + +class TestMeanWaveForms: + run_number = 3798 + max_events = 1 + + def test_mean_waveforms(self): + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = MeanWaveFormsHighLowGain(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/tests/test_pixel_participation.py b/src/nectarchain/dqm/tests/test_pixel_participation.py index 2102478c..4ff33f48 100644 --- a/src/nectarchain/dqm/tests/test_pixel_participation.py +++ b/src/nectarchain/dqm/tests/test_pixel_participation.py @@ -1,30 +1,21 @@ -import os -import sys -import unittest - -import numpy as np from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN from traitlets.config import Config from nectarchain.dqm.pixel_participation import PixelParticipationHighLowGain -script_dir = os.path.dirname(os.path.abspath(__file__)) -dqm_dir = os.path.abspath(os.path.join(script_dir, "../../dqm")) -# Add the directory to sys.path -sys.path.insert(0, dqm_dir) - -class TestPixelParticipation(unittest.TestCase): - run_number = 3938 +class TestPixelParticipation: + run_number = 3798 max_events = 1 def test_pixel_participation(self): - # NectarPath = "/Users/hashkar/Desktop/ashkar_nectar/data/" - # path1 = "NectarCAM.Run3731.0000.fits.fz" - # path = f"{NectarPath}/runs/{path1}" - path1 = "$NECTARDATA/runs/NectarCAM.Run3794.0000.fits.fz" + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 config = None + config = Config( dict( NectarCAMEventSource=dict( @@ -36,12 +27,11 @@ def test_pixel_participation(self): ) ) ) - reader1 = EventSource( - input_url=path1, config=config, max_events=TestPixelParticipation.max_events - ) - Pix, Samp = PixelParticipationHighLowGain.DefineForRun(reader1) - self.assertEqual(np.add(Pix, Samp), 1915) + print(path) + reader1 = EventSource(input_url=path, config=config, max_events=1) -if __name__ == "__main__": - unittest.main() + Pix, Samp = PixelParticipationHighLowGain(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/tests/test_pixel_timeline.py b/src/nectarchain/dqm/tests/test_pixel_timeline.py new file mode 100644 index 00000000..5f5009fe --- /dev/null +++ b/src/nectarchain/dqm/tests/test_pixel_timeline.py @@ -0,0 +1,37 @@ +from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from traitlets.config import Config + +from nectarchain.dqm.pixel_timeline import PixelTimelineHighLowGain + + +class TestPixelTimeline: + run_number = 3798 + max_events = 1 + + def test_pixel_timeline(self): + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = PixelTimelineHighLowGain(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/tests/test_trigger_statistics.py b/src/nectarchain/dqm/tests/test_trigger_statistics.py new file mode 100644 index 00000000..a93341b8 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_trigger_statistics.py @@ -0,0 +1,37 @@ +from ctapipe.io import EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from traitlets.config import Config + +from nectarchain.dqm.trigger_statistics import TriggerStatistics + + +class TestTriggerStatistics: + run_number = 3798 + max_events = 1 + + def test_trigger_statistics(self): + path1 = "/Users/hashkar/Desktop/ashkar_nectar/" + path2 = "data/runs/NectarCAM.Run3798.0000.fits.fz" + path = path1 + path2 + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = TriggerStatistics(HIGH_GAIN).DefineForRun(reader1) + print(Pix, Samp) + # self.assertEqual(Pix + Samp, 1915) + assert Pix + Samp == 1915 diff --git a/src/nectarchain/dqm/trigger_statistics.py b/src/nectarchain/dqm/trigger_statistics.py index fe429223..a33fe0e0 100644 --- a/src/nectarchain/dqm/trigger_statistics.py +++ b/src/nectarchain/dqm/trigger_statistics.py @@ -2,9 +2,10 @@ import numpy as np from astropy import time as astropytime -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class TriggerStatistics(DQMSummary): def __init__(self, gaink): @@ -29,7 +30,6 @@ def __init__(self, gaink): self.TriggerStat_Figures_Dict = {} self.TriggerStat_Figures_Names_Dict = {} - def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix