From e9ca23b5c950ca3dc62a89f3e30074769a956e61 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 17 Jan 2025 19:20:23 +0100 Subject: [PATCH 01/71] move TRR test suite into nectarchain --- environment.yml | 2 +- .../TRR_scripts => trr_test_suite}/README.md | 0 .../src => trr_test_suite}/__init__.py | 0 .../src => trr_test_suite}/deadtime.py | 108 ++++--- .../TRR_scripts => trr_test_suite}/gui.py | 38 ++- .../src => trr_test_suite}/linearity.py | 62 ++-- .../src => trr_test_suite}/pedestal.py | 36 ++- .../pix_couple_tim_uncertainty.py | 51 ++- .../pix_tim_uncertainty.py | 81 +++-- .../tools_components.py | 296 ++++++++++++------ .../src => trr_test_suite}/trigger_timing.py | 41 ++- .../src => trr_test_suite}/utils.py | 80 ++--- 12 files changed, 506 insertions(+), 289 deletions(-) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts => trr_test_suite}/README.md (100%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/__init__.py (100%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/deadtime.py (86%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts => trr_test_suite}/gui.py (97%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/linearity.py (87%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/pedestal.py (82%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/pix_couple_tim_uncertainty.py (71%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/pix_tim_uncertainty.py (78%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/tools_components.py (83%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/trigger_timing.py (81%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/utils.py (88%) diff --git a/environment.yml b/environment.yml index 697f85c6..7d79cf2f 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,7 @@ dependencies: - sphinx - sphinx-automodapi - pydata-sphinx-theme - - pyqt # [linux] + - lmfit - pip: - zeo - zodb diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md b/src/nectarchain/trr_test_suite/README.md similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md rename to src/nectarchain/trr_test_suite/README.md diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py b/src/nectarchain/trr_test_suite/__init__.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py rename to src/nectarchain/trr_test_suite/__init__.py diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py b/src/nectarchain/trr_test_suite/deadtime.py similarity index 86% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py rename to src/nectarchain/trr_test_suite/deadtime.py index 3c2413e4..5c05add3 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py +++ b/src/nectarchain/trr_test_suite/deadtime.py @@ -9,23 +9,29 @@ import numpy as np from astropy import units as u from iminuit import Minuit -from tools_components import DeadtimeTestTool -from utils import ExponentialFitter, deadtime_labels, source_ids_deadtime + +from .tools_components import DeadtimeTestTool +from .utils import ExponentialFitter, deadtime_labels, source_ids_deadtime def get_args(): - """ - Parses command-line arguments for the deadtime test script. + """Parses command-line arguments for the deadtime test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( description="Deadtime tests B-TEL-1260 & B-TEL-1270. \n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween), a corresponding source list and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1000).\n" + + "According to the nectarchain component interface, you have to set a\ + NECTARCAMDATA environment variable in the folder where you have the data\ + from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween), a \ + corresponding source list and an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through \ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed \ + (default 1000).\n" ) parser.add_argument( "-r", @@ -42,7 +48,8 @@ def get_args(): type=int, choices=[0, 1, 2], nargs="+", - help="List of corresponding source for each run: 0 for random generator, 1 for nsb source, 2 for laser", + help="List of corresponding source for each run: 0 for random generator,\ + 1 for nsb source, 2 for laser", required=False, default=source_ids_deadtime, ) @@ -70,15 +77,21 @@ def get_args(): def main(): - """ - Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and B-TEL-1270. + """Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and + B-TEL-1270. - The script takes command-line arguments to specify the list of runs, corresponding sources, number of events to process, and output directory. It then processes the data for each run, performs an exponential fit to the deadtime distribution, and generates two plots: + The script takes command-line arguments to specify the list of runs, corresponding\ + sources, number of events to process, and output directory. It then processes\ + the data for each run, performs an exponential fit to the deadtime\ + distribution, and generates two plots: - 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA requirement indicated. - 2. A plot of the rate from the fit vs. the collected trigger rate, with the relative difference shown in the bottom panel. + 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA\ + requirement indicated. + 2. A plot of the rate from the fit vs. the collected trigger rate, with the\ + relative difference shown in the bottom panel. - The script also saves the generated plots to the specified output directory, and optionally saves them to a temporary output directory for use in a GUI. + The script also saves the generated plots to the specified output directory, and\ + optionally saves them to a temporary output directory for use in a GUI. """ parser = get_args() @@ -189,7 +202,8 @@ def main(): m.limits["deadtime"] = ( 0.6e-6, 1.1e-6, - ) # Put some tigh constrain as the fit will be in trouble when it expect 0. and measured something instead. + ) # Put some tigh constrain as the fit will be in trouble when it expect 0. and + # measured something instead. m.print_level = 2 @@ -202,16 +216,19 @@ def main(): # print(fitted_params_err) print( - f"Dead-Time is {1.e6*fitted_params[1]:.3f} +- {1.e6*fitted_params_err[1]:.3f} µs" + f"Dead-Time is {1.e6*fitted_params[1]:.3f} +- " + f"{1.e6*fitted_params_err[1]:.3f} µs" ) print( - f"Rate is {1./fitted_params[2]:.2f} +- {fitted_params_err[2]/(fitted_params[2]**2):.2f} Hz" + f"Rate is {1./fitted_params[2]:.2f} +-" + f"{fitted_params_err[2]/(fitted_params[2]**2):.2f} Hz" ) print(f"Expected run duration is {fitted_params[0]*fitted_params[2]:.2f} s") fitted_rate.append(1.0 / fitted_params[2]) - # plt.savefig(figurepath + 'deadtime_exponential_fit_nsb_run{}_newfit_cutoff.png'.format(run)) + # plt.savefig(figurepath + 'deadtime_exponential_fit_nsb_run{}_newfit + # _cutoff.png'.format(run)) y = data_content y_fit = fitter.expected_distribution(fitted_params) @@ -236,13 +253,13 @@ def main(): parameter_R2_new_list.append(r2) - deadtime_from_fit = parameter_tau_new_list - deadtime_from_fit_err = parameter_tau_err_new_list + # deadtime_from_fit = parameter_tau_new_list + # deadtime_from_fit_err = parameter_tau_err_new_list lambda_from_fit = parameter_lambda_new_list lambda_from_fit_err = parameter_lambda_err_new_list - A2_from_fit = parameter_A2_new_list - A2_from_fit_err = parameter_A2_err_new_list - R2_from_fit = parameter_R2_new_list + # A2_from_fit = parameter_A2_new_list + # A2_from_fit_err = parameter_A2_err_new_list + # R2_from_fit = parameter_R2_new_list ####################################### # PLOT @@ -260,9 +277,9 @@ def main(): ids = np.array(ids) runlist = np.array(runlist) - ratio_list = [] - collected_rate = [] - err = [] + # ratio_list = [] + # collected_rate = [] + # err = [] for source in range(0, 3): # runl = np.where(ids==source)[0] @@ -399,7 +416,8 @@ def main(): ax1.errorbar( collected_trigger_rate[runl] / 1000, rate[runl], - # xerr=((df_mean_nsb[df_mean_nsb['Run']==run]['Collected_trigger_rate[Hz]_err']))/1000, + # xerr=((df_mean_nsb[df_mean_nsb['Run']==run]['Collected_trigger + # _rate[Hz]_err']))/1000, yerr=rate_err[runl], alpha=0.9, ls=" ", @@ -407,7 +425,8 @@ def main(): color=labels[source]["color"], label=labels[source]["source"], ) - # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']==run]['Voltage[V]'].values[0])) + # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']= + # =run]['Voltage[V]'].values[0])) ax1.legend(frameon=False, prop={"size": 10}, loc="upper left", ncol=1) @@ -424,7 +443,7 @@ def main(): main() -##################################PREVIOUS############################### +# ##################################PREVIOUS############################### # collected_rate = [] @@ -445,8 +464,10 @@ def main(): # for i, run in enumerate(runlist): # deadtime_run, deadtime_bin_run, deadtime_err_run, deadtime_bin_length_run, \ -# total_delta_t_for_busy_time, parameter_A_new, parameter_R_new, parameter_A_err_new, parameter_R_err_new, \ -# first_bin_length, tot_nr_events_histo = deadtime_and_expo_fit(time_tot[i],deadtime_us[i], run) +# total_delta_t_for_busy_time, parameter_A_new, parameter_R_new, parameter_A_err_ +# new, parameter_R_err_new, \ +# first_bin_length, tot_nr_events_histo = deadtime_and_expo_fit(time_tot[i],deadt +# ime_us[i], run) # total_delta_t_for_busy_time_list.append(total_delta_t_for_busy_time) # parameter_A_new_list.append(parameter_A_new) # parameter_R_new_list.append(parameter_R_new) @@ -468,7 +489,8 @@ def main(): # rate_err = (np.array(parameter_R_err_new_list) * 1 / u.us).to(u.kHz).to_value() # A_from_fit = (parameter_A_new_list) # A_from_fit_err = (parameter_A_err_new_list) -# ucts_busy_rate = (np.array(busy_counter[:,-1]) / (np.array(time_tot) * u.s).to(u.s)).to( +# ucts_busy_rate = (np.array(busy_counter[:,-1]) / (np.array(time_tot) * u.s).to(u.s)) +# .to( # u.kHz).value # nr_events_from_histo = (tot_nr_events_histo) # first_bin_delta_t = first_bin_length @@ -478,7 +500,7 @@ def main(): # deadtime_average_err_nsb = np.sqrt(1 / (np.sum(1 / deadtime_bin_length ** 2))) -# ####################################################################################### +# ###################################################################################### # #B-TEL-1260 @@ -502,7 +524,8 @@ def main(): # ratio = rate[i]/freq # ratio_list.append(np.array(ratio)*100) -# ratio_err = np.sqrt((rate_err[i]/freq)**2 + (freq_err*rate[i]/(freq**2))) +# ratio_err = np.sqrt((rate_err[i]/freq)**2 + (freq_err*rate[i]/ +# (freq**2))) # err.append(ratio_err*100) @@ -512,7 +535,8 @@ def main(): # X_sorted = [x for y, x in sorted(zip(Y, X))] # err_sorted = [err for y,err in sorted(zip(Y,err))] -# plt.errorbar(sorted(Y), X_sorted, yerr = err_sorted, alpha=0.6, ls='-', marker='o',color=labels[source]['color'], label = labels[source]['source']) +# plt.errorbar(sorted(Y), X_sorted, yerr = err_sorted, alpha=0.6, ls='-', +# marker='o',color=labels[source]['color'], label = labels[source]['source']) # plt.xlabel('Collected Trigger Rate [kHz]') # plt.ylabel(r'Deadtime [%]') @@ -564,7 +588,8 @@ def main(): # ax1.plot(x, x, color='gray', ls='--', alpha=0.5) # ax2.plot(x, np.zeros(len(x)), color='gray', ls='--', alpha=0.5) -# ax2.fill_between(x, np.ones(len(x))*(-10), np.ones(len(x))*(10), color='gray', alpha=0.1) +# ax2.fill_between(x, np.ones(len(x))*(-10), np.ones(len(x))*(10), color='gray', +# alpha=0.1) # ax2.set_xlabel('Collected Trigger Rate [kHz]') # ax1.set_ylabel(r'Rate from fit [kHz]') @@ -581,11 +606,14 @@ def main(): # #print(collected_triger_rate[runl]) # ax1.errorbar(collected_triger_rate[runl]/1000, # rate[runl], -# #xerr=((df_mean_nsb[df_mean_nsb['Run']==run]['Collected_trigger_rate[Hz]_err']))/1000, +# #xerr=((df_mean_nsb[df_mean_nsb['Run']==run] +# ['Collected_trigger_rate[Hz]_err']))/1000, # yerr=rate_err[runl], # alpha=0.9, -# ls=' ', marker='o', color=labels[source]['color'], label = labels[source]['source']) -# # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']==run]['Voltage[V]'].values[0])) +# ls=' ', marker='o', color=labels[source]['color'], +# label = labels[source]['source']) +# # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']== +# run]['Voltage[V]'].values[0])) # ax1.legend(frameon=False, prop={'size':10}, # loc="upper left", ncol=1) diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py b/src/nectarchain/trr_test_suite/gui.py similarity index 97% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py rename to src/nectarchain/trr_test_suite/gui.py index 686a45a0..cab2fa32 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -1,14 +1,16 @@ -""" -The `TestRunner` class is a GUI application that allows the user to run various tests and display the results. +"""The `TestRunner` class is a GUI application that allows the user to run various tests +and display the results. The class provides the following functionality: - Allows the user to select a test from a dropdown menu. - Dynamically generates input fields based on the selected test. - Runs the selected test and displays the output in a text box. -- Displays the test results in a plot canvas, with navigation buttons to switch between multiple plots. +- Displays the test results in a plot canvas, with navigation buttons to switch between\ + multiple plots. - Provides a dark-themed UI with custom styling for various UI elements. -The class uses the PyQt5 library for the GUI implementation and the Matplotlib library for plotting the test results. +The class uses the PyQt5 library for the GUI implementation and the Matplotlib library\ + for plotting the test results. """ import argparse @@ -17,11 +19,17 @@ import sys import tempfile +import deadtime +import linearity +import pedestal +import pix_couple_tim_uncertainty +import pix_tim_uncertainty +import trigger_timing from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt5.QtCore import QProcess, QTimer -from PyQt5.QtWidgets import ( +from PyQt6.QtCore import QProcess, QTimer +from PyQt6.QtWidgets import ( QApplication, QComboBox, QGroupBox, @@ -44,12 +52,6 @@ sys.path.append(test_dir) # Import test modules -import deadtime -import linearity -import pedestal -import pix_couple_tim_uncertainty -import pix_tim_uncertainty -import trigger_timing class TestRunner(QWidget): @@ -72,7 +74,8 @@ def __init__(self): self.init_ui() def init_ui(self): - # Main layout: vertical, dividing into two sections (top for controls/plot, bottom for output) + # Main layout: vertical, dividing into two sections (top for controls/plot + # , bottom for output) main_layout = QVBoxLayout() self.setStyleSheet( @@ -116,7 +119,8 @@ def init_ui(self): border-radius: 5px; /* Rounded corners */ } QPushButton:disabled { - background-color: rgba(76, 175, 80, 0.5); /* Transparent green when disabled */ + background-color: rgba(76, 175, 80, 0.5); /* Transparent green when\ + disabled */ color: rgba(255, 255, 255, 0.5); /* Light text when disabled */ } QPushButton:hover { @@ -321,11 +325,13 @@ def update_parameters(self): help_button.setToolTip(param_info["help"]) # # Use lambda to capture the current param's help text - # help_button.clicked.connect(lambda _, p=param_info["help"]: self.show_help(p)) + # help_button.clicked.connect(lambda _, p=param_info["help"]: + # self.show_help(p)) # Add the help button to the layout (next to the label) param_layout.addWidget(help_button) - param_layout.addStretch() # Add stretch to push the help button to the right + param_layout.addStretch() # Add stretch to push the help button to + # the right # Add the horizontal layout (label + help button) to the main layout self.param_layout.addLayout(param_layout) diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py b/src/nectarchain/trr_test_suite/linearity.py similarity index 87% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py rename to src/nectarchain/trr_test_suite/linearity.py index ea62de75..b62461d0 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py +++ b/src/nectarchain/trr_test_suite/linearity.py @@ -8,8 +8,9 @@ import matplotlib.pyplot as plt import numpy as np from lmfit.models import Model -from tools_components import LinearityTestTool -from utils import ( + +from .tools_components import LinearityTestTool +from .utils import ( err_ratio, err_sum, linear_fit_function, @@ -19,8 +20,7 @@ def get_args(): - """ - Parses command-line arguments for the linearity test script. + """Parses command-line arguments for the linearity test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. @@ -28,10 +28,18 @@ def get_args(): parser = argparse.ArgumentParser( description="Linearity test B-TEL-1390 & Intensity resolution B-TEL-1010. \n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween), a corresponding transmission list and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 500) and the number of pixels used (default 70).\n" + + "According to the nectarchain component interface, \ + you have to set a NECTARCAMDATA environment variable\ + in the folder where you have the data from your runs\ + or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween), a\ + corresponding transmission list and an output directory to save the \ + final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed\ + (default 500) and the number of pixels used (default 70).\n" ) parser.add_argument( "-r", @@ -75,19 +83,31 @@ def get_args(): def main(): - """ - The `main()` function is the entry point of the linearity test script. It parses the command-line arguments, processes the specified runs, and generates plots to visualize the linearity and charge resolution of the detector. The function performs the following key steps: - - 1. Parses the command-line arguments using the `get_args()` function, which sets up the argument parser and handles the input parameters. - 2. Iterates through the specified run list, processing each run using the `LinearityTestTool` class. This tool initializes, sets up, starts, and finishes the processing for each run, returning the relevant output data. - 3. Normalizes the high-gain and low-gain charge values using the charge value at 0.01 transmission. + """The `main()` function is the entry point of the linearity test script. It parses + the command-line arguments, processes the specified runs, and generates plots to + visualize the linearity and charge resolution of the detector. The function performs + the following key steps: + + 1. Parses the command-line arguments using the `get_args()` function, which sets up\ + the argument parser and handles the input parameters. + 2. Iterates through the specified run list, processing each run using the\ + `LinearityTestTool` class. This tool initializes, sets up, starts, and finishes\ + the processing for each run, returning the relevant output data. + 3. Normalizes the high-gain and low-gain charge values using the charge value at\ + 0.01 transmission. 4. Generates three subplots: - - The first subplot shows the estimated charge vs. the true charge, with the fitted linear function for both high-gain and low-gain channels. - - The second subplot shows the residuals between the estimated and true charge, as a percentage. - - The third subplot shows the ratio of high-gain to low-gain charge, with a fitted linear function. - 5. Saves the generated plots to the specified output directory, and optionally saves temporary plot files for a GUI. - 6. Generates an additional plot to visualize the charge resolution, including the statistical limit. - 7. Saves the charge resolution plot to the specified output directory, and optionally saves a temporary plot file for a GUI. + - The first subplot shows the estimated charge vs. the true charge, with the fitted\ + linear function for both high-gain and low-gain channels. + - The second subplot shows the residuals between the estimated and true charge, as\ + a percentage. + - The third subplot shows the ratio of high-gain to low-gain charge, with a fitted\ + linear function. + 5. Saves the generated plots to the specified output directory, and optionally\ + saves temporary plot files for a GUI. + 6. Generates an additional plot to visualize the charge resolution, including the\ + statistical limit. + 7. Saves the charge resolution plot to the specified output directory, and\ + optionally saves a temporary plot file for a GUI. """ parser = get_args() args = parser.parse_args() @@ -178,7 +198,7 @@ def main(): axs[2].axvspan(10, 1000, alpha=0.2, color="orange") axs[2].set_xlabel("Illumination charge [p.e.]") - for channel, (channel_charge, channel_std, name) in enumerate( + for _, (channel_charge, channel_std, name) in enumerate( zip( [charge_norm_hg, charge_norm_lg], [std_norm_hg, std_norm_lg], diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py b/src/nectarchain/trr_test_suite/pedestal.py similarity index 82% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py rename to src/nectarchain/trr_test_suite/pedestal.py index b20d2fdb..17185a38 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py +++ b/src/nectarchain/trr_test_suite/pedestal.py @@ -7,13 +7,13 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import PedestalTool -from utils import adc_to_pe, pe2photons + +from .tools_components import PedestalTool +from .utils import adc_to_pe, pe2photons def get_args(): - """ - Parses command-line arguments for the linearity test script. + """Parses command-line arguments for the linearity test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. @@ -21,10 +21,17 @@ def get_args(): parser = argparse.ArgumentParser( description="Pedestal substraction test B-TEL-1370.\n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1200).\n" + + "According to the nectarchain component interface, you have to set\ + a NECTARCAMDATA environment variable in the folder where you have\ + the data from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetwee\ + n) and an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be\ + downloaded through DIRAC.\n For the purposes of testing this script,\ + default data is from the runs used for this test in the\ + TRR document.\n" + + "You can optionally specify the number of events to be processed\ + (default 1200).\n" ) parser.add_argument( "-r", @@ -39,7 +46,8 @@ def get_args(): "-e", "--evts", type=int, - help="Number of events to process from each run. Default is 1200. 4000 or more gives best results but takes some time", + help="Number of events to process from each run. Default is 1200. 4000 or more\ + gives best results but takes some time", required=False, default=10, ) @@ -59,13 +67,15 @@ def get_args(): def main(): - """ - The main function that runs the pedestal subtraction test. It parses command-line arguments, processes the specified runs, and generates two plots: + """The main function that runs the pedestal subtraction test. It parses command-line + arguments, processes the specified runs, and generates two plots: 1. A 2D heatmap of the pedestal RMS for all events and pixels. - 2. A line plot of the mean pedestal RMS for each pixel, with the CTA requirement range highlighted. + 2. A line plot of the mean pedestal RMS for each pixel, with the CTA requirement\ + range highlighted. - The function also saves the generated plots to the specified output directory, and optionally saves the first plot to a temporary output file. + The function also saves the generated plots to the specified output directory,\ + and optionally saves the first plot to a temporary output file. """ parser = get_args() diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py similarity index 71% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py rename to src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py index f57f145f..ad599075 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py @@ -9,25 +9,33 @@ def get_args(): - """ - Parses command-line arguments for the pix_couple_tim_uncertainty_test.py script. + """Parses command-line arguments for the pix_couple_tim_uncertainty_test.py script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( - description="Time resolution (timing uncertainty between couples of pixels) test B-TEL-1030.\n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1000). Takes a lot of time.\n" + description="Time resolution (timing uncertainty between couples of pixels)\ + test B-TEL-1030.\n" + + "According to the nectarchain component interface, you have to set a\ + NECTARCAMDATA\ + environment variable in the folder where you have the data from your runs\ + or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween) and\ + an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed (default\ + 1000). Takes a lot of time.\n" ) parser.add_argument( "-r", "--runlist", type=int, nargs="+", - help="List of runs (numbers separated by space). You can put just one run, default 3292", + help="List of runs (numbers separated by space). You can put just one run,\ + default 3292", required=False, default=[3292], ) @@ -35,7 +43,8 @@ def get_args(): "-e", "--evts", type=int, - help="Number of events to process from each run. Default is 100. 1000 or more gives best results but takes some time", + help="Number of events to process from each run. Default is 100. 1000 or\ + more gives best results but takes some time", required=False, default=100, ) @@ -45,7 +54,9 @@ def get_args(): type=str, help=".csv file with pmt transit time corrections", required=False, - default="../transit_time/hv_pmt_tom_correction_laser_measurement_per_pixel_fit_sqrt_hv_newmethod.csv", + default="../transit_time/\ + hv_pmt_tom_correction_laser_measurement_per_pixel_fit_\ + sqrt_hv_newmethod.csv", ) parser.add_argument( "-o", @@ -63,20 +74,26 @@ def get_args(): def main(): - """ - Generates a plot of the RMS of the time-of-maximum (TOM) difference for pairs of pixels, with a visualization of the CTA requirement. + """Generates a plot of the RMS of the time-of-maximum (TOM) difference for pairs of + pixels, with a visualization of the CTA requirement. - The script processes a list of runs, calculates the TOM difference with and without transit time corrections, and plots the distribution of the RMS of the corrected TOM differences. The CTA requirement of 2 ns RMS is visualized on the plot. + The script processes a list of runs, calculates the TOM difference with and without + transit time corrections, and plots the distribution of the RMS of the corrected TOM + differences. The CTA requirement of 2 ns RMS is visualized on the plot. - The script takes several command-line arguments, including the list of runs to process, the number of events to process per run, the path to a CSV file with PMT transit time corrections, and the output directory for the plot. + The script takes several command-line arguments, including the list of runs to + process, the number of events to process per run, the path to a CSV file with PMT + transit time corrections, and the output directory for the plot. - If a temporary output directory is specified, the plot is also saved to a pickle file in that directory for the gui to use. + If a temporary output directory is specified, the plot is also saved to a pickle + file in that directory for the gui to use. """ parser = get_args() args = parser.parse_args() - tt_path = "/Users/dm277349/nectarchain_data/transit_time/hv_pmt_tom_correction_laser_measurement_per_pixel_fit_sqrt_hv_newmethod.csv" + tt_path = "/Users/dm277349/nectarchain_data/transit_time/\ + hv_pmt_tom_correction_laser_measurement_per_pixel_fit_sqrt_hv_newmethod.csv" runlist = args.runlist nevents = args.evts @@ -153,7 +170,7 @@ def main(): arrowprops=dict(color="C4", alpha=0.7, lw=3, arrowstyle="->"), ) - plt.xlabel("RMS of $\Delta t_{\mathrm{TOM}}$ for pairs of pixels [ns]") + plt.xlabel(r"RMS of $\Delta t_{\mathrm{TOM}}$ for pairs of pixels [ns]") plt.ylabel("Normalized entries") plt.gcf() diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py similarity index 78% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py rename to src/nectarchain/trr_test_suite/pix_tim_uncertainty.py index 2c26575b..d4a6c181 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py @@ -12,18 +12,23 @@ def get_args(): - """ - Parses command-line arguments for the pixel timing uncertainty test script. + """Parses command-line arguments for the pixel timing uncertainty test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( description="Systematic pixel timing uncertainty test B-TEL-1380.\n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1200) and the number of pixels used (default 70).\n" + + "According to the nectarchain component interface, you have to set a\ + NECTARCAMDATA environment variable in the folder where you have the data\ + from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween)\ + and an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed (default\ + 1200) and the number of pixels used (default 70).\n" ) parser.add_argument( "-r", @@ -42,7 +47,8 @@ def get_args(): required=False, default=100, ) - # parser.add_argument('-p','--pixels', type = int, help='Number of pixels used. Default is 70', required=False, default=70) + # parser.add_argument('-p','--pixels', type = int, help='Number of pixels used. + # Default is 70', required=False, default=70) parser.add_argument( "-o", "--output", @@ -58,12 +64,14 @@ def get_args(): def main(): - """ - Processes the pixel timing uncertainty test data and generates a plot. + """Processes the pixel timing uncertainty test data and generates a plot. - The function processes the data from the specified list of runs, calculates the weighted mean RMS and RMS error, and generates a plot of the results. The plot is saved to the specified output directory. + The function processes the data from the specified list of runs, calculates the + weighted mean RMS and RMS error, and generates a plot of the results. The plot is + saved to the specified output directory. - If a temporary output directory is provided, the plot is also saved to a pickle file in that directory for the gui to use. + If a temporary output directory is provided, the plot is also saved to a pickle file + in that directory for the gui to use. """ parser = get_args() @@ -129,7 +137,8 @@ def main(): rms_no_fit_weighted_err = [] for run in range(len(runlist)): - # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run])/np.sum(weights_mu_pix[run])) + # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run])/ + # np.sum(weights_mu_pix[run])) # rms_mu_weighted_err.append(np.sqrt(1/np.sum(weights_mu_pix[run]))) rms_no_fit_weighted.append( np.nansum(rms_no_fit[run] * weights_no_fit_pix[run]) @@ -227,15 +236,33 @@ def main(): # from test_tools_components import TimingResolutionTestTool # import argparse -# parser = argparse.ArgumentParser(description='Systematic pixel timing uncertainty test B-TEL-1380.\n' -# +'According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n' -# +'You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n' -# +'If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n' -# +'You can optionally specify the number of events to be processed (default 1200) and the number of pixels used (default 70).\n') -# parser.add_argument('-r','--runlist', type = int, nargs='+', help='List of runs (numbers separated by space)', required=False) -# parser.add_argument('-e','--evts', type = int, help='Number of events to process from each run. Default is 1200. 4000 or more gives best results but takes some time', required=False, default=1200) -# #parser.add_argument('-p','--pixels', type = int, help='Number of pixels used. Default is 70', required=False, default=70) -# parser.add_argument('-o','--output', type=str, help='Output directory. If none, plot will be saved in current directory', required=False, default='./') +# parser = argparse.ArgumentParser(description='Sy +# stematic pixel timing uncertainty test B-TEL-1380.\n' +# +'According to +# the nectarchain component interface, you have to set a NECTARCAMDATA environment +# variable in the folder where you have the data from your runs or where you want them +# to be downloaded.\n' +# +'You have to g +# ive a list of runs (run numbers with spaces inbetween) and an output directory to save +# the final plot.\n' +# +'If the data i +# s not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes +# of testing this script, default data is from the runs used for this test in the TRR +# document.\n' +# +'You can optio +# nally specify the number of events to be processed (default 1200) and the +# number of +# pixels used (default 70).\n') +# parser.add_argument('-r','--runlist', type = int +# , nargs='+', help='List of runs (numbers separated by space)', required=False) +# parser.add_argument('-e','--evts', type = int, h +# elp='Number of events to process from each run. Default is 1200. 4000 or more gives +# best results but takes some time', required=False, default=1200) +# #parser.add_argument('-p','--pixels', type = int +# , help='Number of pixels used. Default is 70', required=False, default=70) +# parser.add_argument('-o','--output', type=str, h +# elp='Output directory. If none, plot will be saved in current directory', +# required=False, default='./') # args = parser.parse_args() @@ -257,7 +284,8 @@ def main(): # for run in runlist: # print("PROCESSING RUN {}".format(run)) # tool = TimingResolutionTestTool( -# progress_bar=True, run_number=run, max_events=nevents, log_level=20, window_width=16, overwrite=True +# progress_bar=True, run_number=run, max_events=nevents, log_level=20, +# window_width=16, overwrite=True # ) # tool.initialize() # tool.setup() @@ -301,9 +329,11 @@ def main(): # rms_no_fit_weighted_err = [] # for run in range(len(runlist)): -# # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run])/np.sum(weights_mu_pix[run])) +# # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run]) +# /np.sum(weights_mu_pix[run])) # # rms_mu_weighted_err.append(np.sqrt(1/np.sum(weights_mu_pix[run]))) -# rms_no_fit_weighted.append(np.nansum(rms_no_fit[run]*weights_no_fit_pix[run])/np.nansum(weights_no_fit_pix[run])) +# rms_no_fit_weighted.append(np.nansum(rms_no_fit[run]*weights_no_fit_pix[run]) +# /np.nansum(weights_no_fit_pix[run])) # rms_no_fit_weighted_err.append(np.sqrt(1/np.nansum(weights_no_fit_pix[run]))) @@ -325,7 +355,8 @@ def main(): # plt.axhline(1, ls='--', color='C4', alpha=0.6) -# plt.axhline(1/np.sqrt(12), ls='--', color='gray', alpha=0.7, label='Quantification rms noise') +# plt.axhline(1/np.sqrt(12), ls='--', color='gray', alpha=0.7, label= +# 'Quantification rms noise') # plt.axvspan(20, 1000, alpha=0.1, color='C4') diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py b/src/nectarchain/trr_test_suite/tools_components.py similarity index 83% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py rename to src/nectarchain/trr_test_suite/tools_components.py index a1a4c52e..158ce4c6 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py +++ b/src/nectarchain/trr_test_suite/tools_components.py @@ -21,12 +21,16 @@ # overriding so we can have maxevents in the path def _init_output_path(self): - """ - Initializes the output path for the NectarCAMCalibrationTool. + """Initializes the output path for the NectarCAMCalibrationTool. - If `max_events` is `None`, the output file name will be in the format `{self.name}_run{self.run_number}.h5`. Otherwise, the file name will be in the format `{self.name}_run{self.run_number}_maxevents{self.max_events}.h5`. + If `max_events` is `None`, the output file name will be in the format\ + `{self.name}_run{self.run_number}.h5`. Otherwise, the file name will\ + be in the format\ + `{self.name}_run{self.run_number}_maxevents{self.max_events}.h5`. - The output path is constructed by joining the `NECTARCAMDATA` environment variable (or `/tmp` if not set) with the `tests` subdirectory and the generated file name. + The output path is constructed by joining the `NECTARCAMDATA` environment variable\ + (or `/tmp` if not set) with the `tests` subdirectory and the generated\ + file name. """ if self.max_events is None: @@ -42,8 +46,8 @@ def _init_output_path(self): class ChargeContainer(NectarCAMContainer): - """ - This class contains fields that store various properties and data related to NectarCAM events, including: + """This class contains fields that store various properties and data related to + NectarCAM events, including: - `run_number`: The run number associated with the waveforms. - `npixels`: The number of effective pixels. @@ -81,15 +85,22 @@ class ChargeContainer(NectarCAMContainer): class ChargeComp(NectarCAMComponent): - """ - This class `ChargeComp` is a NectarCAMComponent that processes NectarCAM event data. It extracts the charge information from the waveforms of each event, handling cases of saturated or noisy events. The class has the following configurable parameters: + """This class `ChargeComp` is a NectarCAMComponent that processes NectarCAM event + data. It extracts the charge information from the waveforms of each event, handling + cases of saturated or noisy events. The class has the following configurable + parameters: - `window_shift`: The time in ns before the peak to extract the charge. - `window_width`: The duration of the charge extraction window in ns. - The `__init__` method initializes important members of the component, such as timestamps, event type, event ids, pedestal and charge for both gain channels. - The `__call__` method is the main processing logic, which is called for each event. It extracts the charge information for both high gain and low gain channels, handling various cases such as saturated events and events with no signal. - The `finish` method collects all the processed data and returns a `ChargeContainer` object containing the run number, number of pixels, pixel IDs, UCTS timestamps, event types, event IDs, and the high and low gain charge values. + The `__init__` method initializes important members of the component, such as\ + timestamps, event type, event ids, pedestal and charge for both gain channels. + The `__call__` method is the main processing logic, which is called for each event.\ + It extracts the charge information for both high gain and low gain channels,\ + handling various cases such as saturated events and events with no signal. + The `finish` method collects all the processed data and returns a `ChargeContainer`\ + object containing the run number, number of pixels, pixel IDs, UCTS timestamps,\ + event types, event IDs, and the high and low gain charge values. """ window_shift = Integer( @@ -106,7 +117,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain + # interesting quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -118,7 +130,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__charge_hg = [] self.__charge_lg = [] - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__event_id.append(np.uint32(event.index.event_id)) @@ -131,7 +143,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): wfs.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id]) wfs.append(event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id]) - #####THE JOB IS HERE###### + # ###THE JOB IS HERE#### for i, (pedestal, charge) in enumerate( zip( [self.__pedestal_hg, self.__pedestal_lg], @@ -181,9 +193,11 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): integral[pix] = 0 else: - # x = np.linspace(0,signal_stop[pix]-signal_start[pix],signal_stop[pix]-signal_start[pix]) + # x = np.linspace(0,signal_stop[pix]-signal_start[pix], + # signal_stop[pix]-signal_start[pix]) # spl = UnivariateSpline(x,y) - # integral[pix] = spl.integral(0,signal_stop[pix]-signal_start[pix]) + # integral[pix] = spl.integral(0,signal_stop[pix]- + # signal_start[pix]) integral[pix] = np.sum( wf[pix, signal_start[pix] : signal_stop[pix]] @@ -193,7 +207,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): charge.append(chg) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = ChargeContainer( run_number=ChargeContainer.fields["run_number"].type(self._run_number), @@ -218,10 +232,16 @@ def finish(self): class LinearityTestTool(EventsLoopNectarCAMCalibrationTool): - """ - This class, `LinearityTestTool`, is a subclass of `EventsLoopNectarCAMCalibrationTool`. It is responsible for performing a linearity test on NectarCAM data. The class has a `componentsList` attribute that specifies the list of NectarCAM components to be applied. - - The `finish` method is the main functionality of this class. It reads the charge data from the output file, calculates the mean charge, standard deviation, and standard error for both the high gain and low gain channels, and returns these values. This information can be used to assess the linearity of the NectarCAM system. + """This class, `LinearityTestTool`, is a subclass of + `EventsLoopNectarCAMCalibrationTool`. It is responsible for performing a linearity + test on NectarCAM data. The class has a `componentsList` attribute that specifies + the list of NectarCAM components to be applied. + + The `finish` method is the main functionality of this class. It reads the charge\ + data from the output file, calculates the mean charge, standard deviation,\ + and standard error for both the high gain and low gain channels, and\ + returns these values. This information can be used to assess\ + the linearity of the NectarCAM system. """ name = "LinearityTestTool" @@ -255,7 +275,7 @@ def finish(self, *args, **kwargs): charge_hg.extend(tup[6]) charge_lg.extend(tup[7]) - except: + except Exception: break output_file.close() @@ -288,7 +308,8 @@ class ToMContainer(NectarCAMContainer): event_type (np.ndarray[np.uint8]): The trigger event types. event_id (np.ndarray[np.uint32]): The event IDs. charge_hg (np.ndarray[np.float64]): The mean high gain charge per event. - tom_no_fit (np.ndarray[np.float64]): The time of maximum from the data (no fitting). + tom_no_fit (np.ndarray[np.float64]): The time of maximum from\ + the data (no fitting). good_evts (np.ndarray[np.uint32]): The IDs of the good (non-cosmic ray) events. """ @@ -317,11 +338,13 @@ class ToMContainer(NectarCAMContainer): ) # tom_mu = Field( - # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of maximum of signal fitted with gaussian" + # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of maximum of + # signal fitted with gaussian" # ) # tom_sigma = Field( - # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of fitted maximum sigma" + # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of fitted + # maximum sigma" # ) tom_no_fit = Field( type=np.ndarray, @@ -338,14 +361,25 @@ class ToMContainer(NectarCAMContainer): class ToMComp(NectarCAMComponent): - """ - This class, `ToMComp`, is a component of the NectarCAM system that is responsible for processing waveform data. It has several configurable parameters, including the width and shift before the peak of the time window for charge extraction, the peak height threshold. - - The `__init__` method initializes some important component members, such as timestamps, event type, event ids, pedestal and charge values for both gain channels. - - The `__call__` method is the main entry point for processing an event. It extracts the waveform data, calculates the pedestal, charge, and time of maximum (ToM) for each pixel, and filters out events that do not meet the peak height threshold. The results are stored in various member variables, which are then returned in the `finish` method. - - The `finish` method collects the processed data from the member variables and returns a `ToMContainer` object, which contains the run number, number of pixels, pixel IDs, UCTS timestamps, event types, event IDs, high-gain charge, ToM without fitting, and IDs of good (non-cosmic ray) events. + """This class, `ToMComp`, is a component of the NectarCAM system that is responsible + for processing waveform data. It has several configurable parameters, including the + width and shift before the peak of the time window for charge extraction, the peak + height threshold. + + The `__init__` method initializes some important component members, such as\ + timestamps, event type, event ids, pedestal and charge values for both gain\ + channels. + + The `__call__` method is the main entry point for processing an event. It extracts\ + the waveform data, calculates the pedestal, charge, and time of maximum (ToM)\ + for each pixel, and filters out events that do not meet the peak\ + height threshold. The results are stored in various member variables,\ + which are then returned in the `finish` method. + + The `finish` method collects the processed data from the member variables and\ + returns a `ToMContainer` object, which contains the run number, number of\ + pixels, pixel IDs, UCTS timestamps, event types, event IDs, high-gain\ + charge, ToM without fitting, and IDs of good (non-cosmic ray) events. """ window_shift = Integer( @@ -367,7 +401,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain + # interesting quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -386,7 +421,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__ff_event_ind = -1 - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__event_id.append(np.uint32(event.index.event_id)) @@ -398,7 +433,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__ff_event_ind += 1 - #####THE JOB IS HERE###### + # #####THE JOB IS HERE###### for i, (pedestal, charge, tom_no_fit) in enumerate( zip([self.__pedestal_hg], [self.__charge_hg], [self.__tom_no_fit]) @@ -483,7 +518,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # # fit # model = Model(gaus) - # params = model.make_params(a=yi[peaks[max_peak_index]] * 3, mu=mean, sigma=sigma) + # params = model.make_params(a=yi[peaks[max_peak_index]] * 3, + # mu=mean, sigma=sigma) # result = model.fit(y_fit, params, x=x_fit) # result_sigma = result.params['sigma'].value @@ -510,12 +546,16 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # change_grad_pos_left = 3 # change_grad_pos_right = 3 # mean = xi[peaks[max_peak_index]] - # sigma = change_grad_pos_right + change_grad_pos_left # define window for the gaussian fit + # sigma = change_grad_pos_right + change_grad_pos_left # define + # window for the gaussian fit - # x_fit = xi[peaks[max_peak_index]-change_grad_pos_left:peaks[max_peak_index]+change_grad_pos_right] - # y_fit = yi[peaks[max_peak_index]-change_grad_pos_left:peaks[max_peak_index]+change_grad_pos_right] + # x_fit = xi[peaks[max_peak_index]-change_grad_pos_left:peaks + # [max_peak_index]+change_grad_pos_right] + # y_fit = yi[peaks[max_peak_index]-change_grad_pos_left:peaks + # [max_peak_index]+change_grad_pos_right] # model = Model(gaus) - # params = model.make_params(a=yi[peaks[max_peak_index]],mu=mean,sigma=sigma) + # params = model.make_params(a=yi[peaks[max_peak_index]], + # mu=mean,sigma=sigma) # result = model.fit(y_fit, params, x=x_fit) max_position_x_prefit = xi[peaks[max_peak_index]] @@ -523,7 +563,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # result_mu = result.params['mu'].value else: - # index_x_window_min = list(xi).index(closest_value(xi, signal_start[pix])) + # index_x_window_min = list(xi).index(closest_value(xi, + # signal_start[pix])) charge_sum = y[ signal_start[pix] : signal_start[pix] + self.window_width ].sum() @@ -533,10 +574,12 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # result_mu = -1 else: - # If no maximum is found, the integration is done between 20 and 36 ns. + # If no maximum is found, the integration is done between 20 and 36 + # ns. signal_start[pix] = 20 - # index_x_window_min = list(xi).index(closest_value(xi, signal_start[pix])) + # index_x_window_min = list(xi).index(closest_value(xi, + # signal_start[pix])) charge_sum = y[ signal_start[pix] : signal_start[pix] + self.window_width ].sum() @@ -562,7 +605,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # print("is good evt") self.__good_evts.append(self.__ff_event_ind) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = ToMContainer( run_number=ToMContainer.fields["run_number"].type(self._run_number), @@ -587,12 +630,20 @@ def finish(self): class TimingResolutionTestTool(EventsLoopNectarCAMCalibrationTool): - """ - This class, `TimingResolutionTestTool`, is a subclass of `EventsLoopNectarCAMCalibrationTool` and is used to perform timing resolution tests on NectarCAM data. It reads the output data from the `ToMContainer` dataset and processes the charge, timing, and event information to calculate the timing resolution and mean charge in photoelectrons. - - The `finish()` method is the main entry point for this tool. It reads the output data from the HDF5 file, filters the data to remove cosmic ray events, and then calculates the timing resolution and mean charge per photoelectron. The timing resolution is calculated using a weighted mean and variance approach, with an option to use a bootstrapping method to estimate the error on the RMS value. - - The method returns the RMS of the timing resolution, the error on the RMS, and the mean charge in photoelectrons. + """This class, `TimingResolutionTestTool`, is a subclass of + `EventsLoopNectarCAMCalibrationTool` and is used to perform timing resolution tests + on NectarCAM data. It reads the output data from the `ToMContainer` dataset and + processes the charge, timing, and event information to calculate the timing + resolution and mean charge in photoelectrons. + + The `finish()` method is the main entry point for this tool. It reads the output + data from the HDF5 file, filters the data to remove cosmic ray events, and then + calculates the timing resolution and mean charge per photoelectron. The timing + resolution is calculated using a weighted mean and variance approach, with an option + to use a bootstrapping method to estimate the error on the RMS value. + + The method returns the RMS of the timing resolution, the error on the RMS, and the + mean charge in photoelectrons. """ name = "TimingResolutionTestTool" @@ -625,7 +676,7 @@ def finish(self, bootstrap=False, *args, **kwargs): charge_all.extend(tup[6]) tom_no_fit_all.extend(tup[7]) good_evts.extend(tup[8]) - except: + except Exception: break output_file.close() @@ -640,9 +691,11 @@ def finish(self, bootstrap=False, *args, **kwargs): # print(good_evts) charge = charge_all[good_evts] mean_charge_pe = np.mean(np.mean(charge, axis=0)) / 58.0 - # tom_mu = np.array(tom_mu_all[good_evts]).reshape(len(good_evts),output[0].npixels) + # tom_mu = np.array(tom_mu_all[good_evts]).reshape(len(good_evts), + # output[0].npixels) - # tom_sigma = np.array(tom_sigma_all[good_evts]).reshape(len(good_evts),output[0].npixels) + # tom_sigma = np.array(tom_sigma_all[good_evts]).reshape(len(good_evts), + # output[0].npixels) tom_no_fit = np.array(tom_no_fit_all[good_evts]).reshape( len(good_evts), npixels ) @@ -679,7 +732,8 @@ def finish(self, bootstrap=False, *args, **kwargs): bootsample = np.random.choice( sample, size=int(3 / 4 * (len(sample))), replace=True ) - # print(len(bootsample), bootsample.mean(), bootsample.std()) + # print(len(bootsample), bootsample.mean(), + # bootsample.std()) boot_rms.append(bootsample.std()) # simulated mean of rms bootrms_mean = np.mean(boot_rms) @@ -707,14 +761,15 @@ def finish(self, bootstrap=False, *args, **kwargs): rms[pix] = np.sqrt(weighted_variance) # print("RMS:", rms[pix]) - # Compute the total number of data points (sum of histogram values, i.e. N) + # Compute the total number of data points (sum of histogram + # values, i.e. N) N = np.sum(hist_values) # print("Total number of events (N):", N) # Error on the standard deviation err[pix] = rms[pix] / np.sqrt(2 * N) # print("Error on RMS:", err[pix]) - except: + except Exception: # no data rms[pix] = np.nan err[pix] = np.nan @@ -723,17 +778,22 @@ def finish(self, bootstrap=False, *args, **kwargs): class ToMPairsTool(EventsLoopNectarCAMCalibrationTool): - """ - This class, `ToMPairsTool`, is an `EventsLoopNectarCAMCalibrationTool` that is used to process ToM (Time of maximum) data from NectarCAM. + """This class, `ToMPairsTool`, is an `EventsLoopNectarCAMCalibrationTool`\ + that is used to process ToM (Time of maximum) data from NectarCAM. The `finish` method has the following functionalities: - - It reads in ToM data from an HDF5 file and applies a transit time correction to the ToM values using a provided lookup table. - - It calculates the time difference between ToM pairs for both corrected and uncorrected ToM values. - - It returns the uncorrected ToM values, the corrected ToM values, the pixel IDs, and the time difference calculations for the uncorrected and corrected ToM values. - - The class has several configurable parameters, including the list of NectarCAM components to apply, the maximum number of events to process, and the output file path. - + - It reads in ToM data from an HDF5 file and applies a transit time correction to\ + the ToM values using a provided lookup table. + - It calculates the time difference between ToM pairs for both corrected and\ + uncorrected ToM values. + - It returns the uncorrected ToM values, the corrected ToM values, the pixel IDs,\ + and the time difference calculations for the uncorrected and corrected\ + ToM values. + + The class has several configurable parameters, including the list of NectarCAM\ + components to apply, the maximum number of events to process, and the output\ + file path. """ name = "ToMPairsTool" @@ -765,7 +825,7 @@ def finish(self, *args, **kwargs): try: pixels_id.extend(tup[2]) tom_no_fit_all.extend(tup[7]) - except: + except Exception: break output_file.close() @@ -851,8 +911,8 @@ def finish(self, *args, **kwargs): class PedestalContainer(NectarCAMContainer): - """ - Attributes of the PedestalContainer class that store various data related to the pedestal of a NectarCAM event. + """Attributes of the PedestalContainer class that store various data related to the + pedestal of a NectarCAM event. Attributes: run_number (np.uint16): The run number associated with the waveforms. @@ -914,21 +974,30 @@ class PedestalContainer(NectarCAMContainer): class PedestalComp(NectarCAMComponent): - """ - The `PedestalComp` class is a NectarCAMComponent that is responsible for processing the pedestal and RMS of the high and low gain waveforms for each event. - - The `__init__` method initializes the `PedestalComp` class. It sets up several member variables to store pedestal related data such as timestamps, event types, event IDs, pedestal and pedestal rms values for both gains. - - The `__call__` method is called for each event, and it processes the waveforms to calculate the pedestal and RMS for the high and low gain channels. The results are stored in the class attributes `__pedestal_hg`, `__pedestal_lg`, `__rms_ped_hg`, and `__rms_ped_lg`. - - The `finish` method is called at the end of processing, and it returns a `PedestalContainer` object containing the calculated pedestal and RMS values, as well as other event information. + """The `PedestalComp` class is a NectarCAMComponent that is responsible for + processing the pedestal and RMS of the high and low gain waveforms for each event. + + The `__init__` method initializes the `PedestalComp` class. It sets up several\ + member variables to store pedestal related data such as timestamps,\ + event types,\ + event IDs, pedestal and pedestal rms values for both gains. + + The `__call__` method is called for each event, and it processes the waveforms to\ + calculate the pedestal and RMS for the high and low gain channels. The results\ + are stored in the class attributes `__pedestal_hg`, `__pedestal_lg`, \ + `__rms_ped_hg`, and `__rms_ped_lg`. + + The `finish` method is called at the end of processing, and it returns a\ + `PedestalContainer` object containing the calculated pedestal and RMS values\ + , as well as other event information. """ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain interesting + # quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -941,7 +1010,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__rms_ped_hg = [] self.__rms_ped_lg = [] - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__event_id.append(np.uint32(event.index.event_id)) @@ -954,7 +1023,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): wfs.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id]) wfs.append(event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id]) - #####THE JOB IS HERE###### + # #####THE JOB IS HERE###### for i, (pedestal, rms_pedestal) in enumerate( zip( @@ -972,7 +1041,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): pedestal.append(ped) rms_pedestal.append(rms_ped) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = PedestalContainer( run_number=PedestalContainer.fields["run_number"].type(self._run_number), @@ -1002,12 +1071,16 @@ def finish(self): class PedestalTool(EventsLoopNectarCAMCalibrationTool): - """ - This class is a part of the PedestalTool, which is an EventsLoopNectarCAMCalibrationTool. + """This class is a part of the PedestalTool, which is an + EventsLoopNectarCAMCalibrationTool. - The finish() method opens the output file, which is an HDF5 file, and extracts the `rms_ped_hg` (root mean square of the high gain pedestal) values from the `PedestalContainer` dataset. Finally, it closes the output file and returns the list of `rms_ped_hg` values. + The finish() method opens the output file, which is an HDF5 file,\ + and extracts the `rms_ped_hg` (root mean square of the high gain pedestal)\ + values from the `PedestalContainer` dataset. Finally, it closes the output\ + file and returns the list of `rms_ped_hg` values. - This method is used to post-process the output of the PedestalTool and extract specific information from the generated HDF5 file. + This method is used to post-process the output of the PedestalTool and extract\ + specific information from the generated HDF5 file. """ name = "PedestalTool" @@ -1042,7 +1115,7 @@ def finish(self, *args, **kwargs): for tup in data: try: rms_ped_hg.extend(tup[8]) - except: + except Exception: break output_file.close() @@ -1051,8 +1124,8 @@ def finish(self, *args, **kwargs): class UCTSContainer(NectarCAMContainer): - """ - Defines the fields for the UCTSContainer class, which is used to store various data related to UCTS events. + """Defines the fields for the UCTSContainer class, which is used to store various + data related to UCTS events. The fields include: - `run_number`: The run number associated with the waveforms. @@ -1096,12 +1169,15 @@ class UCTSContainer(NectarCAMContainer): class UCTSComp(NectarCAMComponent): - """ - The `__init__` method initializes the `UCTSComp` class, which is a NectarCAMComponent. It sets up several member variables to store UCTS related data, such as timestamps, event types, event IDs, busy counters, and event counters. + """The `__init__` method initializes the `UCTSComp` class, which is a + NectarCAMComponent. It sets up several member variables to store UCTS related data, + such as timestamps, event types, event IDs, busy counters, and event counters. - The `__call__` method is called for each event, and it appends the UCTS-related data from the event to the corresponding member variables. + The `__call__` method is called for each event, and it appends the UCTS-related\ + data from the event to the corresponding member variables. - The `finish` method creates and returns a `UCTSContainer` object, which is a container for the UCTS-related data that was collected during the event loop. + The `finish` method creates and returns a `UCTSContainer` object, which is a\ + container for the UCTS-related data that was collected during the event loop. """ window_shift = Integer( @@ -1120,7 +1196,8 @@ def __init__( super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain interesting + # quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -1130,7 +1207,7 @@ def __init__( self.excl_muons = None self.__mean_event_charge = [] - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): take_event = True @@ -1193,7 +1270,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): if self.excl_muons: self.__mean_event_charge.append(mean_charge) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = UCTSContainer( run_number=UCTSContainer.fields["run_number"].type(self._run_number), @@ -1218,10 +1295,16 @@ def finish(self): class DeadtimeTestTool(EventsLoopNectarCAMCalibrationTool): - """ - The `DeadtimeTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that is used to test the deadtime of NectarCAM. - - The `finish` method is responsible for reading the data from the HDF5 file, extracting the relevant information (UCTS timestamps, event counters, and busy counters), and calculating the deadtime-related metrics. The method returns the UCTS timestamps, the time differences between consecutive UCTS timestamps, the event counters, the busy counters, the collected trigger rate, the total time, and the deadtime percentage. + """The `DeadtimeTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that is + used to test the deadtime of NectarCAM. + + The `finish` method is responsible for reading the data from the HDF5 file,\ + extracting the relevant information (UCTS timestamps, event counters, and\ + busy counters), and calculating the deadtime-related metrics. The method\ + returns the UCTS timestamps, the time differences between consecutive\ + UCTS timestamps, the event counters, the busy counters,\ + the collected\ + trigger rate, the total time, and the deadtime percentage. """ name = "DeadtimeTestTool" @@ -1250,7 +1333,7 @@ def finish(self, *args, **kwargs): ucts_timestamps.extend(tup[3]) event_counter.extend(tup[7]) busy_counter.extend(tup[6]) - except: + except Exception: break # print(output_file.keys()) # tom_mu_all= output[0].tom_mu @@ -1285,10 +1368,15 @@ def finish(self, *args, **kwargs): class TriggerTimingTestTool(EventsLoopNectarCAMCalibrationTool): - """ - The `TriggerTimingTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that is used to test the trigger timing of NectarCAM. - - The `finish` method is responsible for reading the data from the HDF5 file, extracting the relevant information (UCTS timestamps), and calculating the RMS value of the difference between consecutive triggers. The method returns the UCTS timestamps, the time differences between consecutive triggers for events concerning more than 10 pixels (non-muon related events). + """The `TriggerTimingTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that + is used to test the trigger timing of NectarCAM. + + The `finish` method is responsible for reading the data from the HDF5 file,\ + extracting the relevant information (UCTS timestamps), and calculating\ + the RMS value of the difference between consecutive triggers. The method\ + returns the UCTS timestamps, the time differences between consecutive\ + triggers for events concerning more than 10 pixels (non-muon\ + related events). """ name = "TriggerTimingTestTool" @@ -1323,7 +1411,7 @@ def finish(self, *args, **kwargs): ucts_timestamps.extend(tup[3]) charge_per_event.extend(tup[4]) - except: + except Exception: break # print(output_file.keys()) # tom_mu_all= output[0].tom_mu diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py b/src/nectarchain/trr_test_suite/trigger_timing.py similarity index 81% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py rename to src/nectarchain/trr_test_suite/trigger_timing.py index 6dcc8f7d..0d679959 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py +++ b/src/nectarchain/trr_test_suite/trigger_timing.py @@ -12,18 +12,23 @@ def get_args(): - """ - Parses command-line arguments for the deadtime test script. + """Parses command-line arguments for the deadtime test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( description="Trigger Timing Test B-TEL-1410. \n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1000).\n" + + "According to the nectarchain component interface, you have to set\ + a NECTARCAMDATA environment variable in the folder where you have the data\ + from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween) and an\ + output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed\ + (default 1000).\n" ) parser.add_argument( "-r", @@ -58,15 +63,21 @@ def get_args(): def main(): - """ - Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and B-TEL-1270. + """Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and + B-TEL-1270. - The script takes command-line arguments to specify the list of runs, corresponding sources, number of events to process, and output directory. It then processes the data for each run, performs an exponential fit to the deadtime distribution, and generates two plots: + The script takes command-line arguments to specify the list of runs, corresponding\ + sources, number of events to process, and output directory. It then processes\ + the data for each run, performs an exponential fit to the deadtime\ + distribution, and generates two plots: - 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA requirement indicated. - 2. A plot of the rate from the fit vs. the collected trigger rate, with the relative difference shown in the bottom panel. + 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA\ + requirement indicated. + 2. A plot of the rate from the fit vs. the collected trigger rate, with the\ + relative difference shown in the bottom panel. - The script also saves the generated plots to the specified output directory, and optionally saves them to a temporary output directory for use in a GUI. + The script also saves the generated plots to the specified output directory,\ + and optionally saves them to a temporary output directory for use in a GUI. """ parser = get_args() @@ -141,7 +152,8 @@ def main(): cta_requirement_y = 5 # Y-value for the CTA requirement ax.axhline(y=cta_requirement_y, color="purple", linestyle="--") - # Add the small vertical arrows starting from the CTA requirement line and pointing downwards + # Add the small vertical arrows starting from the CTA requirement line and pointing + # downwards arrow_positions = [20, 80, 200] # X-positions for the arrows for x_pos in arrow_positions: ax.annotate( @@ -151,7 +163,8 @@ def main(): arrowprops=dict(arrowstyle="->", color="purple", lw=1.5), ) # Arrow pointing downwards - # Add the CTA requirement label exactly above the dashed line, centered between arrows + # Add the CTA requirement label exactly above the dashed line, centered between + # arrows ax.text( 140, cta_requirement_y + 0.5, diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py b/src/nectarchain/trr_test_suite/utils.py similarity index 88% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py rename to src/nectarchain/trr_test_suite/utils.py index 27f8b202..b8fec4c9 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py +++ b/src/nectarchain/trr_test_suite/utils.py @@ -241,13 +241,15 @@ def pe_from_intensity_percentage( percent_from_calibration=intensity_percent, known_charge=intensity_to_charge, ): - """ - Converts a percentage of intensity to the corresponding charge value based on a known calibration. + """Converts a percentage of intensity to the corresponding charge value based on a + known calibration. Args: percent (numpy.ndarray): The percentage of intensity to convert to charge. - percent_from_calibration (numpy.ndarray, optional): The known percentages of intensity used in the calibration. Defaults to `intensity_percent`. - known_charge (numpy.ndarray, optional): The known charge values corresponding to the calibration percentages. Defaults to `intensity_to_charge`. + percent_from_calibration (numpy.ndarray, optional): The known percentages of\ + intensity used in the calibration. Defaults to `intensity_percent`. + known_charge (numpy.ndarray, optional): The known charge values corresponding\ + to the calibration percentages. Defaults to `intensity_to_charge`. Returns: numpy.ndarray: The charge values corresponding to the input percentages. @@ -265,8 +267,7 @@ def pe_from_intensity_percentage( # functions by federica def linear_fit_function(x, a, b): - """ - Computes a linear function of the form `a*x + b`. + """Computes a linear function of the form `a*x + b`. Args: x (float): The input value. @@ -280,8 +281,7 @@ def linear_fit_function(x, a, b): def second_degree_fit_function(x, a, b, c): - """ - Computes a quadratic function of the form `a*(x**2) + b*x + c`. + """Computes a quadratic function of the form `a*(x**2) + b*x + c`. Args: x (float): The input value. @@ -296,8 +296,7 @@ def second_degree_fit_function(x, a, b, c): def third_degree_fit_function(x, a, b, c, d): - """ - Computes a function of the form `(a*x + b)/(1+c) + d`. + """Computes a function of the form `(a*x + b)/(1+c) + d`. Args: x (float): The input value. @@ -313,8 +312,7 @@ def third_degree_fit_function(x, a, b, c, d): def fit_function_hv(x, a, b): - """ - Computes a function of the form `a/sqrt(x) + b`. + """Computes a function of the form `a/sqrt(x) + b`. Args: x (float): The input value. @@ -328,15 +326,16 @@ def fit_function_hv(x, a, b): def err_ratio(nominator, denominator, err_norm, err_denom, cov_nom_den=0): - """ - Computes the error ratio for a given nominator, denominator, and their respective errors. + """Computes the error ratio for a given nominator, denominator, and their respective + errors. Args: nominator (float): The nominator value. denominator (float): The denominator value. err_norm (float): The error of the nominator. err_denom (float): The error of the denominator. - cov_nom_den (float, optional): The covariance between the nominator and denominator. Defaults to 0. + cov_nom_den (float, optional): The covariance between the nominator and\ + denominator. Defaults to 0. Returns: float: The error ratio. @@ -351,10 +350,11 @@ def err_ratio(nominator, denominator, err_norm, err_denom, cov_nom_den=0): def err_sum(err_a, err_b, cov_a_b=0): - """ - Computes the square root of the sum of the squares of `err_a` and `err_b`, plus twice the covariance `cov_a_b`. + """Computes the square root of the sum of the squares of `err_a` and `err_b`, plus + twice the covariance `cov_a_b`. - This function is used to calculate the combined error of two values, taking into account their individual errors and the covariance between them. + This function is used to calculate the combined error of two values, taking into\ + account their individual errors and the covariance between them. Args: err_a (float): The error of the first value. @@ -369,18 +369,22 @@ def err_sum(err_a, err_b, cov_a_b=0): # from stackoverflow def argmedian(x, axis=None): - """ - Returns the index of the median element in the input array `x` along the specified axis. + """Returns the index of the median element in the input array `x` along the + specified axis. - If `axis` is `None`, the function returns the index of the median element in the flattened array. - Otherwise, it computes the argmedian along the specified axis and returns an array of indices. + If `axis` is `None`, the function returns the index of the median element in\ + the flattened array. + Otherwise, it computes the argmedian along the specified axis and returns an\ + array of indices. Args: x (numpy.ndarray): The input array. - axis (int or None, optional): The axis along which to compute the argmedian. If `None`, the argmedian is computed on the flattened array. + axis (int or None, optional): The axis along which to compute the argmedian.\ + If `None`, the argmedian is computed on the flattened array. Returns: - int or numpy.ndarray: The index or indices of the median element(s) in the input array. + int or numpy.ndarray: The index or indices of the median element(s) in the\ + input array. """ if axis is None: return np.argpartition(x, len(x) // 2)[len(x) // 2] @@ -392,8 +396,8 @@ def argmedian(x, axis=None): def pe2photons(x): - """ - Converts the input value `x` from photons to photoelectrons (PE) by multiplying it by 4. + """Converts the input value `x` from photons to photoelectrons (PE) by multiplying + it by 4. Args: x (float): The input value in photons. @@ -405,8 +409,8 @@ def pe2photons(x): def photons2pe(x): - """ - Converts the input value `x` from photoelectrons (PE) to photons by dividing it by 4. + """Converts the input value `x` from photoelectrons (PE) to photons by dividing it + by 4. Args: x (float): The input value in photoelectrons. @@ -419,8 +423,8 @@ def photons2pe(x): # from federica's notebook class ExponentialFitter: - """ - Represents an exponential fitter class that computes the expected distribution and the minus 2 log likelihood for a given dataset and exponential parameters. + """Represents an exponential fitter class that computes the expected distribution + and the minus 2 log likelihood for a given dataset and exponential parameters. Attributes: datas (numpy.ndarray): The input data array. @@ -428,7 +432,8 @@ class ExponentialFitter: Methods: compute_expected_distribution(norm, loc, scale): - Computes the expected distribution given the normalization, location, and scale parameters. + Computes the expected distribution given the normalization, location, and\ + scale parameters. expected_distribution(x): Returns the expected distribution given the parameters in `x`. compute_minus2loglike(x): @@ -466,8 +471,7 @@ def compute_minus2loglike(self, x): def pois(x, A, R): - """ - Computes the expected distribution for a Poisson process with rate parameter `R`. + """Computes the expected distribution for a Poisson process with rate parameter `R`. Args: x (float): The input value. @@ -477,13 +481,12 @@ def pois(x, A, R): Returns: float: The expected distribution for the Poisson process. """ - """poisson function, parameter r (rate) is the fit parameter""" + """Poisson function, parameter r (rate) is the fit parameter.""" return A * np.exp(x * R) def deadtime_and_expo_fit(time_tot, deadtime_us, run, output_plot=None): - """ - Computes the deadtime and exponential fit parameters for a given dataset. + """Computes the deadtime and exponential fit parameters for a given dataset. Args: time_tot (float): The total time of the dataset. @@ -579,8 +582,9 @@ def deadtime_and_expo_fit(time_tot, deadtime_us, run, output_plot=None): ax.text( 600, entries[1] / 1, - "$y = A \cdot \exp({-R \cdot x})$\n" - # + r'$A=%2.2f \pm %2.2f$'%(as_si((result.params['A'].value/1000)*1e3,2), as_si((result.params['A'].stderr/1000)*1e3,2)) + r"$y = A \cdot \exp({-R \cdot x})$\n" + # + r'$A=%2.2f \pm %2.2f$'%(as_si((result.params['A'].value/1000) + # *1e3,2), as_si((result.params['A'].stderr/1000)*1e3,2)) + r"$A=%2.2f \pm %2.2f$" % (result.params["A"].value, result.params["A"].stderr) + "\n" From 2877409b850b0fdcbc5ee3b74efb134cb46e4d86 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 20 Jan 2025 20:31:23 +0100 Subject: [PATCH 02/71] fix imports --- src/nectarchain/trr_test_suite/__init__.py | 5 +++++ src/nectarchain/trr_test_suite/gui.py | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/trr_test_suite/__init__.py b/src/nectarchain/trr_test_suite/__init__.py index e69de29b..417f98f8 100644 --- a/src/nectarchain/trr_test_suite/__init__.py +++ b/src/nectarchain/trr_test_suite/__init__.py @@ -0,0 +1,5 @@ +from .gui import TestRunner + +__all__ = [ + "TestRunner", +] diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index cab2fa32..92ce0f63 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -19,17 +19,11 @@ import sys import tempfile -import deadtime -import linearity -import pedestal -import pix_couple_tim_uncertainty -import pix_tim_uncertainty -import trigger_timing from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt6.QtCore import QProcess, QTimer -from PyQt6.QtWidgets import ( +from PyQt5.QtCore import QProcess, QTimer +from PyQt5.QtWidgets import ( QApplication, QComboBox, QGroupBox, @@ -46,6 +40,15 @@ QWidgetItem, ) +import nectarchain.trr_test_suite.deadtime as deadtime +import nectarchain.trr_test_suite.linearity as linearity +import nectarchain.trr_test_suite.pedestal as pedestal +import nectarchain.trr_test_suite.pix_tim_uncertainty as pix_tim_uncertainty +import nectarchain.trr_test_suite.trigger_timing as trigger_timing +from nectarchain.trr_test_suite import ( + pix_couple_tim_uncertainty as pix_couple_tim_uncertainty, +) + # Ensure the src directory is in sys.path test_dir = os.path.abspath("src") if test_dir not in sys.path: From bc8890afda3bb0b5df2d1d54ad96e7aa0a661f29 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 22 Jan 2025 15:44:01 +0100 Subject: [PATCH 03/71] fix imports and format gange pyq5 to pyqt6 --- src/nectarchain/trr_test_suite/deadtime.py | 8 ++++++-- src/nectarchain/trr_test_suite/gui.py | 16 ++++++++++------ src/nectarchain/trr_test_suite/linearity.py | 4 ++-- src/nectarchain/trr_test_suite/pedestal.py | 4 ++-- .../trr_test_suite/pix_couple_tim_uncertainty.py | 9 +++++---- .../trr_test_suite/pix_tim_uncertainty.py | 5 +++-- .../trr_test_suite/tools_components.py | 2 +- src/nectarchain/trr_test_suite/trigger_timing.py | 5 +++-- 8 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/nectarchain/trr_test_suite/deadtime.py b/src/nectarchain/trr_test_suite/deadtime.py index 5c05add3..c01a5c27 100644 --- a/src/nectarchain/trr_test_suite/deadtime.py +++ b/src/nectarchain/trr_test_suite/deadtime.py @@ -10,8 +10,12 @@ from astropy import units as u from iminuit import Minuit -from .tools_components import DeadtimeTestTool -from .utils import ExponentialFitter, deadtime_labels, source_ids_deadtime +from nectarchain.trr_test_suite.tools_components import DeadtimeTestTool +from nectarchain.trr_test_suite.utils import ( + ExponentialFitter, + deadtime_labels, + source_ids_deadtime, +) def get_args(): diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 92ce0f63..d49c89aa 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -19,11 +19,11 @@ import sys import tempfile -from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar +from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt5.QtCore import QProcess, QTimer -from PyQt5.QtWidgets import ( +from PyQt6.QtCore import QProcess, QTimer +from PyQt6.QtWidgets import ( QApplication, QComboBox, QGroupBox, @@ -182,7 +182,9 @@ def init_ui(self): # Add a stretchable spacer to push the canvas to the right top_layout.addSpacerItem( - QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum) + QSpacerItem( + 40, 20, QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Minimum + ) ) # Create a vertical layout for the plot container @@ -405,7 +407,9 @@ def run_test(self): self.output_text_edit.clear() self.process = QProcess(self) - self.process.setProcessChannelMode(QProcess.MergedChannels) + self.process.setProcessChannelMode( + QProcess.ProcessChannelMode.MergedChannels + ) self.process.readyReadStandardOutput.connect(self.read_process_output) self.process.finished.connect(self.process_finished) @@ -540,4 +544,4 @@ def cleanup_tempdir(self): if __name__ == "__main__": app = QApplication(sys.argv) ex = TestRunner() - sys.exit(app.exec_()) + sys.exit(app.exec()) diff --git a/src/nectarchain/trr_test_suite/linearity.py b/src/nectarchain/trr_test_suite/linearity.py index b62461d0..1d05c8d2 100644 --- a/src/nectarchain/trr_test_suite/linearity.py +++ b/src/nectarchain/trr_test_suite/linearity.py @@ -9,8 +9,8 @@ import numpy as np from lmfit.models import Model -from .tools_components import LinearityTestTool -from .utils import ( +from nectarchain.trr_test_suite.tools_components import LinearityTestTool +from nectarchain.trr_test_suite.utils import ( err_ratio, err_sum, linear_fit_function, diff --git a/src/nectarchain/trr_test_suite/pedestal.py b/src/nectarchain/trr_test_suite/pedestal.py index 17185a38..58abc9eb 100644 --- a/src/nectarchain/trr_test_suite/pedestal.py +++ b/src/nectarchain/trr_test_suite/pedestal.py @@ -8,8 +8,8 @@ import matplotlib.pyplot as plt import numpy as np -from .tools_components import PedestalTool -from .utils import adc_to_pe, pe2photons +from nectarchain.trr_test_suite.tools_components import PedestalTool +from nectarchain.trr_test_suite.utils import adc_to_pe, pe2photons def get_args(): diff --git a/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py index ad599075..2138ae06 100644 --- a/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py @@ -5,7 +5,8 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import ToMPairsTool + +from nectarchain.trr_test_suite.tools_components import ToMPairsTool def get_args(): @@ -54,9 +55,9 @@ def get_args(): type=str, help=".csv file with pmt transit time corrections", required=False, - default="../transit_time/\ - hv_pmt_tom_correction_laser_measurement_per_pixel_fit_\ - sqrt_hv_newmethod.csv", + default="../transit_time/" + "hv_pmt_tom_correction_laser_measurement_per_pixel_fit" + "sqrt_hv_newmethod.csv", ) parser.add_argument( "-o", diff --git a/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py index d4a6c181..53c644c1 100644 --- a/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py @@ -7,8 +7,9 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import TimingResolutionTestTool -from utils import pe2photons, photons2pe + +from nectarchain.trr_test_suite.tools_components import TimingResolutionTestTool +from nectarchain.trr_test_suite.utils import pe2photons, photons2pe def get_args(): diff --git a/src/nectarchain/trr_test_suite/tools_components.py b/src/nectarchain/trr_test_suite/tools_components.py index 158ce4c6..5154c93d 100644 --- a/src/nectarchain/trr_test_suite/tools_components.py +++ b/src/nectarchain/trr_test_suite/tools_components.py @@ -12,11 +12,11 @@ from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from scipy.interpolate import InterpolatedUnivariateSpline from scipy.signal import find_peaks -from utils import adc_to_pe, argmedian from nectarchain.data.container import NectarCAMContainer from nectarchain.makers import EventsLoopNectarCAMCalibrationTool from nectarchain.makers.component import NectarCAMComponent +from nectarchain.trr_test_suite.utils import adc_to_pe, argmedian # overriding so we can have maxevents in the path diff --git a/src/nectarchain/trr_test_suite/trigger_timing.py b/src/nectarchain/trr_test_suite/trigger_timing.py index 0d679959..933cf03e 100644 --- a/src/nectarchain/trr_test_suite/trigger_timing.py +++ b/src/nectarchain/trr_test_suite/trigger_timing.py @@ -7,8 +7,9 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import TriggerTimingTestTool -from utils import pe2photons + +from nectarchain.trr_test_suite.tools_components import TriggerTimingTestTool +from nectarchain.trr_test_suite.utils import pe2photons def get_args(): From 8f7796977cd10b4384936cb31a1eb0ddab13cb7c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 22 Jan 2025 15:50:48 +0100 Subject: [PATCH 04/71] add h5py depencie --- environment.yml | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 7d79cf2f..6240c9e1 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,9 @@ dependencies: - sphinx - sphinx-automodapi - pydata-sphinx-theme - - lmfit + - lmfit # needed into TRR + - h5py # needed into TRR (should be removed to use I/O methods of containers) + - pyqt # [linux] - pip: - zeo - zodb diff --git a/pyproject.toml b/pyproject.toml index d5eb3148..bafbc220 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "scipy==1.11.4", "zodb", "zeo", + "h5py", "pyqt6", "pyqtgraph", ] From a8514b5f29c6f33a74f8d7fd1f400cced714e3a2 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 22 Jan 2025 15:54:51 +0100 Subject: [PATCH 05/71] add lmfit to pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index bafbc220..79deaba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "scipy==1.11.4", "zodb", "zeo", + "lmfit", "h5py", "pyqt6", "pyqtgraph", From 226a1dd3184d3e1e3b6f5f0dd6e3d89735d45555 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 00:16:03 +0100 Subject: [PATCH 06/71] fix docstring --- src/nectarchain/trr_test_suite/gui.py | 8 +++----- src/nectarchain/trr_test_suite/linearity.py | 10 +++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index d49c89aa..7e42298e 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -1,6 +1,5 @@ -"""The `TestRunner` class is a GUI application that allows the user to run various tests -and display the results. - +"""The ``TestRunner`` class is a GUI application that allows the\ + user to run various tests and display the results. The class provides the following functionality: - Allows the user to select a test from a dropdown menu. - Dynamically generates input fields based on the selected test. @@ -8,7 +7,6 @@ - Displays the test results in a plot canvas, with navigation buttons to switch between\ multiple plots. - Provides a dark-themed UI with custom styling for various UI elements. - The class uses the PyQt5 library for the GUI implementation and the Matplotlib library\ for plotting the test results. """ @@ -129,7 +127,7 @@ def init_ui(self): QPushButton:hover { background-color: #45a049; /* Darker green on hover */ } - """ + """ ) # Horizontal layout for test options (left) and plot canvas (right) diff --git a/src/nectarchain/trr_test_suite/linearity.py b/src/nectarchain/trr_test_suite/linearity.py index 1d05c8d2..ab4a6cd5 100644 --- a/src/nectarchain/trr_test_suite/linearity.py +++ b/src/nectarchain/trr_test_suite/linearity.py @@ -83,11 +83,11 @@ def get_args(): def main(): - """The `main()` function is the entry point of the linearity test script. It parses - the command-line arguments, processes the specified runs, and generates plots to - visualize the linearity and charge resolution of the detector. The function performs - the following key steps: - + """ + The `main()` function is the entry point of the linearity test script. It parses \ + the command-line arguments, processes the specified runs, and generates plots\ + to visualize the linearity and charge resolution of the detector. The\ + function performs the following key steps: 1. Parses the command-line arguments using the `get_args()` function, which sets up\ the argument parser and handles the input parameters. 2. Iterates through the specified run list, processing each run using the\ From 12964ede9e5e13ab87a8722a7a18c6ba193a1862 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 17:06:10 +0100 Subject: [PATCH 07/71] fix sphinx issue with PyQt6 subclass imports --- src/nectarchain/trr_test_suite/gui.py | 67 +++++++++++++++------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 7e42298e..6cd16170 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -1,16 +1,3 @@ -"""The ``TestRunner`` class is a GUI application that allows the\ - user to run various tests and display the results. -The class provides the following functionality: -- Allows the user to select a test from a dropdown menu. -- Dynamically generates input fields based on the selected test. -- Runs the selected test and displays the output in a text box. -- Displays the test results in a plot canvas, with navigation buttons to switch between\ - multiple plots. -- Provides a dark-themed UI with custom styling for various UI elements. -The class uses the PyQt5 library for the GUI implementation and the Matplotlib library\ - for plotting the test results. -""" - import argparse import os import pickle @@ -20,23 +7,30 @@ from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt6.QtCore import QProcess, QTimer -from PyQt6.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, -) + +if "sphinx" not in sys.modules: + from PyQt6.QtCore import QProcess, QTimer + from PyQt6.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, + ) +else: + + class QWidget: + pass + import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity @@ -56,6 +50,19 @@ class TestRunner(QWidget): + """The ``TestRunner`` class is a GUI application that allows the\ + user to run various tests and display the results. + The class provides the following functionality: + - Allows the user to select a test from a dropdown menu. + - Dynamically generates input fields based on the selected test. + - Runs the selected test and displays the output in a text box. + - Displays the test results in a plot canvas, with navigation buttons\ + to switch between multiple plots. + - Provides a dark-themed UI with custom styling for various UI elements. + The class uses the PyQt5 library for the GUI implementation and the Matplotlib\ + library for plotting the test results. + """ + test_modules = { "Linearity Test": linearity, "Deadtime Test": deadtime, From ce5762dc9b47f9175d4285d881257a65e4f4bac2 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:02:39 +0100 Subject: [PATCH 08/71] Revert "Merge branch 'main' into TTR_tests" This reverts commit d07747ab1b0b976358ad4cd9621891c1f24b699b, reversing changes made to 2dadf7e04b6b6d2ec37f008cf42792c41fa7433c. --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79192908..3adeca1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,10 +144,7 @@ jobs: run: | pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts - - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - uses: codecov/codecov-action@v5 docs: From 59fd5b12cbab333b5324070c00e7ba557a91092f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:39:08 +0100 Subject: [PATCH 09/71] realign to main --- environment.yml | 2 +- .../makers/component/tests/test_flatfield_spe_component.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 6240c9e1..6706e6b0 100644 --- a/environment.yml +++ b/environment.yml @@ -25,5 +25,5 @@ dependencies: - zodb - mechanize - browser-cookie3 + - pyqt6 - pyqtgraph - - pyqt6 # [osx and arm64] diff --git a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py index bb5eae67..fb0a8aff 100644 --- a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py +++ b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py @@ -71,7 +71,6 @@ def test_call(self, instance, event): instance(event) assert len(instance.chargesComponent.trigger_list) == 1 - # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): SPEalgorithm.window_length.default_value = 2 # We need to re-instance objects because otherwise, en exception is raised : @@ -188,7 +187,6 @@ def test_init(self, instance): assert isinstance(instance.chargesComponent, ChargesComponent) assert instance._chargesContainers is None - # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): # We need to re-instance objects because otherwise, en exception is raised : # ReferenceError('weakly-referenced object no longer exists') From 5e461d60299934fa306060822332e23c8871b613 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:40:41 +0100 Subject: [PATCH 10/71] fix to main --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6706e6b0..6240c9e1 100644 --- a/environment.yml +++ b/environment.yml @@ -25,5 +25,5 @@ dependencies: - zodb - mechanize - browser-cookie3 - - pyqt6 - pyqtgraph + - pyqt6 # [osx and arm64] From c61ba24c0279a393d1b7c50754844bee695494ae Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:52:02 +0100 Subject: [PATCH 11/71] reparing unwanted changes --- .github/workflows/ci.yml | 5 ++++- .../makers/component/tests/test_flatfield_spe_component.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3adeca1a..79192908 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,7 +144,10 @@ jobs: run: | pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts - - uses: codecov/codecov-action@v5 + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v5 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} docs: diff --git a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py index fb0a8aff..bb5eae67 100644 --- a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py +++ b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py @@ -71,6 +71,7 @@ def test_call(self, instance, event): instance(event) assert len(instance.chargesComponent.trigger_list) == 1 + # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): SPEalgorithm.window_length.default_value = 2 # We need to re-instance objects because otherwise, en exception is raised : @@ -187,6 +188,7 @@ def test_init(self, instance): assert isinstance(instance.chargesComponent, ChargesComponent) assert instance._chargesContainers is None + # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): # We need to re-instance objects because otherwise, en exception is raised : # ReferenceError('weakly-referenced object no longer exists') From c04c4b5a273849d04bd6e3c5ba0661d06b73eb94 Mon Sep 17 00:00:00 2001 From: jlenain Date: Thu, 30 Jan 2025 22:25:45 +0100 Subject: [PATCH 12/71] Fix Sphinx intermapping with PyQt documentation --- docs/conf.py | 1 + environment.yml | 1 + pyproject.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 891807ad..e420d667 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,6 +51,7 @@ # class signature) "numpydoc", "sphinx_design", + "sphinx_qt_documentation", ] autosummary_generate = True # Turn on sphinx.ext.autosummary diff --git a/environment.yml b/environment.yml index 6240c9e1..feb99e09 100644 --- a/environment.yml +++ b/environment.yml @@ -27,3 +27,4 @@ dependencies: - browser-cookie3 - pyqtgraph - pyqt6 # [osx and arm64] + - sphinx-qt-documentation diff --git a/pyproject.toml b/pyproject.toml index 79deaba9..5a930987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ docs = [ "sphinx-autodoc-typehints", "sphinx-automodapi", "sphinx-design", + "sphinx-qt-documentation", "pydata_sphinx_theme", "numpydoc", "tomli; python_version < '3.11'" From eeef61351cd90dd5264d03128591a2a141e92839 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 31 Jan 2025 10:51:07 +0100 Subject: [PATCH 13/71] remove import fix for sphinx --- src/nectarchain/trr_test_suite/gui.py | 41 +++++++++++---------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 6cd16170..99feff31 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -7,30 +7,23 @@ from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure - -if "sphinx" not in sys.modules: - from PyQt6.QtCore import QProcess, QTimer - from PyQt6.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, - ) -else: - - class QWidget: - pass - +from PyQt6.QtCore import QProcess, QTimer +from PyQt6.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, +) import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity From 0d2848953b2bd471a00f16defd02345c502156cf Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 31 Jan 2025 14:43:30 +0100 Subject: [PATCH 14/71] Switch back to PyQt5 instead of PyQt6, to be able to launch the TRR GUI, on Linux at least --- pyproject.toml | 3 ++- src/nectarchain/trr_test_suite/gui.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5a930987..4e3124e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ dependencies = [ "zeo", "lmfit", "h5py", - "pyqt6", + 'pyqt5 ; platform_system == "Linux"', + 'pyqt6 ; platform_system != "Linux"', # for macOS "pyqtgraph", ] diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 99feff31..1dea5516 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -4,11 +4,11 @@ import sys import tempfile -from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar +from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt6.QtCore import QProcess, QTimer -from PyQt6.QtWidgets import ( +from PyQt5.QtCore import QProcess, QTimer +from PyQt5.QtWidgets import ( QApplication, QComboBox, QGroupBox, From f9f0e39fdcc878fc6cf288bbfba187038f81a20b Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 31 Jan 2025 16:36:36 +0100 Subject: [PATCH 15/71] To try to ensure that the TRR GUI is launchable both from Linux and macOS, add conditional imports on PyQt5/PyQt6. --- src/nectarchain/trr_test_suite/gui.py | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 1dea5516..36789451 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -4,26 +4,49 @@ import sys import tempfile -from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt5.QtCore import QProcess, QTimer -from PyQt5.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, -) + +try: + from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar + from PyQt6.QtCore import QProcess, QTimer + from PyQt6.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, + ) +except ImportError: + from PyQt5.QtCore import QProcess, QTimer + from PyQt5.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, + ) + from matplotlib.backends.backend_qt5 import ( + NavigationToolbar2QT as NavigationToolbar, + ) import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity From 462ebf215a527a45360140d8a552fd14020225a7 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 17 Jan 2025 19:20:23 +0100 Subject: [PATCH 16/71] move TRR test suite into nectarchain --- environment.yml | 2 +- .../TRR_scripts => trr_test_suite}/README.md | 0 .../src => trr_test_suite}/__init__.py | 0 .../src => trr_test_suite}/deadtime.py | 108 ++++--- .../TRR_scripts => trr_test_suite}/gui.py | 38 ++- .../src => trr_test_suite}/linearity.py | 62 ++-- .../src => trr_test_suite}/pedestal.py | 36 ++- .../pix_couple_tim_uncertainty.py | 51 ++- .../pix_tim_uncertainty.py | 81 +++-- .../tools_components.py | 296 ++++++++++++------ .../src => trr_test_suite}/trigger_timing.py | 41 ++- .../src => trr_test_suite}/utils.py | 80 ++--- 12 files changed, 506 insertions(+), 289 deletions(-) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts => trr_test_suite}/README.md (100%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/__init__.py (100%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/deadtime.py (86%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts => trr_test_suite}/gui.py (97%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/linearity.py (87%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/pedestal.py (82%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/pix_couple_tim_uncertainty.py (71%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/pix_tim_uncertainty.py (78%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/tools_components.py (83%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/trigger_timing.py (81%) rename src/nectarchain/{user_scripts/dmousadi/TRR_scripts/src => trr_test_suite}/utils.py (88%) diff --git a/environment.yml b/environment.yml index 697f85c6..7d79cf2f 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,7 @@ dependencies: - sphinx - sphinx-automodapi - pydata-sphinx-theme - - pyqt # [linux] + - lmfit - pip: - zeo - zodb diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md b/src/nectarchain/trr_test_suite/README.md similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md rename to src/nectarchain/trr_test_suite/README.md diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py b/src/nectarchain/trr_test_suite/__init__.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py rename to src/nectarchain/trr_test_suite/__init__.py diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py b/src/nectarchain/trr_test_suite/deadtime.py similarity index 86% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py rename to src/nectarchain/trr_test_suite/deadtime.py index 3c2413e4..5c05add3 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py +++ b/src/nectarchain/trr_test_suite/deadtime.py @@ -9,23 +9,29 @@ import numpy as np from astropy import units as u from iminuit import Minuit -from tools_components import DeadtimeTestTool -from utils import ExponentialFitter, deadtime_labels, source_ids_deadtime + +from .tools_components import DeadtimeTestTool +from .utils import ExponentialFitter, deadtime_labels, source_ids_deadtime def get_args(): - """ - Parses command-line arguments for the deadtime test script. + """Parses command-line arguments for the deadtime test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( description="Deadtime tests B-TEL-1260 & B-TEL-1270. \n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween), a corresponding source list and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1000).\n" + + "According to the nectarchain component interface, you have to set a\ + NECTARCAMDATA environment variable in the folder where you have the data\ + from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween), a \ + corresponding source list and an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through \ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed \ + (default 1000).\n" ) parser.add_argument( "-r", @@ -42,7 +48,8 @@ def get_args(): type=int, choices=[0, 1, 2], nargs="+", - help="List of corresponding source for each run: 0 for random generator, 1 for nsb source, 2 for laser", + help="List of corresponding source for each run: 0 for random generator,\ + 1 for nsb source, 2 for laser", required=False, default=source_ids_deadtime, ) @@ -70,15 +77,21 @@ def get_args(): def main(): - """ - Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and B-TEL-1270. + """Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and + B-TEL-1270. - The script takes command-line arguments to specify the list of runs, corresponding sources, number of events to process, and output directory. It then processes the data for each run, performs an exponential fit to the deadtime distribution, and generates two plots: + The script takes command-line arguments to specify the list of runs, corresponding\ + sources, number of events to process, and output directory. It then processes\ + the data for each run, performs an exponential fit to the deadtime\ + distribution, and generates two plots: - 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA requirement indicated. - 2. A plot of the rate from the fit vs. the collected trigger rate, with the relative difference shown in the bottom panel. + 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA\ + requirement indicated. + 2. A plot of the rate from the fit vs. the collected trigger rate, with the\ + relative difference shown in the bottom panel. - The script also saves the generated plots to the specified output directory, and optionally saves them to a temporary output directory for use in a GUI. + The script also saves the generated plots to the specified output directory, and\ + optionally saves them to a temporary output directory for use in a GUI. """ parser = get_args() @@ -189,7 +202,8 @@ def main(): m.limits["deadtime"] = ( 0.6e-6, 1.1e-6, - ) # Put some tigh constrain as the fit will be in trouble when it expect 0. and measured something instead. + ) # Put some tigh constrain as the fit will be in trouble when it expect 0. and + # measured something instead. m.print_level = 2 @@ -202,16 +216,19 @@ def main(): # print(fitted_params_err) print( - f"Dead-Time is {1.e6*fitted_params[1]:.3f} +- {1.e6*fitted_params_err[1]:.3f} µs" + f"Dead-Time is {1.e6*fitted_params[1]:.3f} +- " + f"{1.e6*fitted_params_err[1]:.3f} µs" ) print( - f"Rate is {1./fitted_params[2]:.2f} +- {fitted_params_err[2]/(fitted_params[2]**2):.2f} Hz" + f"Rate is {1./fitted_params[2]:.2f} +-" + f"{fitted_params_err[2]/(fitted_params[2]**2):.2f} Hz" ) print(f"Expected run duration is {fitted_params[0]*fitted_params[2]:.2f} s") fitted_rate.append(1.0 / fitted_params[2]) - # plt.savefig(figurepath + 'deadtime_exponential_fit_nsb_run{}_newfit_cutoff.png'.format(run)) + # plt.savefig(figurepath + 'deadtime_exponential_fit_nsb_run{}_newfit + # _cutoff.png'.format(run)) y = data_content y_fit = fitter.expected_distribution(fitted_params) @@ -236,13 +253,13 @@ def main(): parameter_R2_new_list.append(r2) - deadtime_from_fit = parameter_tau_new_list - deadtime_from_fit_err = parameter_tau_err_new_list + # deadtime_from_fit = parameter_tau_new_list + # deadtime_from_fit_err = parameter_tau_err_new_list lambda_from_fit = parameter_lambda_new_list lambda_from_fit_err = parameter_lambda_err_new_list - A2_from_fit = parameter_A2_new_list - A2_from_fit_err = parameter_A2_err_new_list - R2_from_fit = parameter_R2_new_list + # A2_from_fit = parameter_A2_new_list + # A2_from_fit_err = parameter_A2_err_new_list + # R2_from_fit = parameter_R2_new_list ####################################### # PLOT @@ -260,9 +277,9 @@ def main(): ids = np.array(ids) runlist = np.array(runlist) - ratio_list = [] - collected_rate = [] - err = [] + # ratio_list = [] + # collected_rate = [] + # err = [] for source in range(0, 3): # runl = np.where(ids==source)[0] @@ -399,7 +416,8 @@ def main(): ax1.errorbar( collected_trigger_rate[runl] / 1000, rate[runl], - # xerr=((df_mean_nsb[df_mean_nsb['Run']==run]['Collected_trigger_rate[Hz]_err']))/1000, + # xerr=((df_mean_nsb[df_mean_nsb['Run']==run]['Collected_trigger + # _rate[Hz]_err']))/1000, yerr=rate_err[runl], alpha=0.9, ls=" ", @@ -407,7 +425,8 @@ def main(): color=labels[source]["color"], label=labels[source]["source"], ) - # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']==run]['Voltage[V]'].values[0])) + # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']= + # =run]['Voltage[V]'].values[0])) ax1.legend(frameon=False, prop={"size": 10}, loc="upper left", ncol=1) @@ -424,7 +443,7 @@ def main(): main() -##################################PREVIOUS############################### +# ##################################PREVIOUS############################### # collected_rate = [] @@ -445,8 +464,10 @@ def main(): # for i, run in enumerate(runlist): # deadtime_run, deadtime_bin_run, deadtime_err_run, deadtime_bin_length_run, \ -# total_delta_t_for_busy_time, parameter_A_new, parameter_R_new, parameter_A_err_new, parameter_R_err_new, \ -# first_bin_length, tot_nr_events_histo = deadtime_and_expo_fit(time_tot[i],deadtime_us[i], run) +# total_delta_t_for_busy_time, parameter_A_new, parameter_R_new, parameter_A_err_ +# new, parameter_R_err_new, \ +# first_bin_length, tot_nr_events_histo = deadtime_and_expo_fit(time_tot[i],deadt +# ime_us[i], run) # total_delta_t_for_busy_time_list.append(total_delta_t_for_busy_time) # parameter_A_new_list.append(parameter_A_new) # parameter_R_new_list.append(parameter_R_new) @@ -468,7 +489,8 @@ def main(): # rate_err = (np.array(parameter_R_err_new_list) * 1 / u.us).to(u.kHz).to_value() # A_from_fit = (parameter_A_new_list) # A_from_fit_err = (parameter_A_err_new_list) -# ucts_busy_rate = (np.array(busy_counter[:,-1]) / (np.array(time_tot) * u.s).to(u.s)).to( +# ucts_busy_rate = (np.array(busy_counter[:,-1]) / (np.array(time_tot) * u.s).to(u.s)) +# .to( # u.kHz).value # nr_events_from_histo = (tot_nr_events_histo) # first_bin_delta_t = first_bin_length @@ -478,7 +500,7 @@ def main(): # deadtime_average_err_nsb = np.sqrt(1 / (np.sum(1 / deadtime_bin_length ** 2))) -# ####################################################################################### +# ###################################################################################### # #B-TEL-1260 @@ -502,7 +524,8 @@ def main(): # ratio = rate[i]/freq # ratio_list.append(np.array(ratio)*100) -# ratio_err = np.sqrt((rate_err[i]/freq)**2 + (freq_err*rate[i]/(freq**2))) +# ratio_err = np.sqrt((rate_err[i]/freq)**2 + (freq_err*rate[i]/ +# (freq**2))) # err.append(ratio_err*100) @@ -512,7 +535,8 @@ def main(): # X_sorted = [x for y, x in sorted(zip(Y, X))] # err_sorted = [err for y,err in sorted(zip(Y,err))] -# plt.errorbar(sorted(Y), X_sorted, yerr = err_sorted, alpha=0.6, ls='-', marker='o',color=labels[source]['color'], label = labels[source]['source']) +# plt.errorbar(sorted(Y), X_sorted, yerr = err_sorted, alpha=0.6, ls='-', +# marker='o',color=labels[source]['color'], label = labels[source]['source']) # plt.xlabel('Collected Trigger Rate [kHz]') # plt.ylabel(r'Deadtime [%]') @@ -564,7 +588,8 @@ def main(): # ax1.plot(x, x, color='gray', ls='--', alpha=0.5) # ax2.plot(x, np.zeros(len(x)), color='gray', ls='--', alpha=0.5) -# ax2.fill_between(x, np.ones(len(x))*(-10), np.ones(len(x))*(10), color='gray', alpha=0.1) +# ax2.fill_between(x, np.ones(len(x))*(-10), np.ones(len(x))*(10), color='gray', +# alpha=0.1) # ax2.set_xlabel('Collected Trigger Rate [kHz]') # ax1.set_ylabel(r'Rate from fit [kHz]') @@ -581,11 +606,14 @@ def main(): # #print(collected_triger_rate[runl]) # ax1.errorbar(collected_triger_rate[runl]/1000, # rate[runl], -# #xerr=((df_mean_nsb[df_mean_nsb['Run']==run]['Collected_trigger_rate[Hz]_err']))/1000, +# #xerr=((df_mean_nsb[df_mean_nsb['Run']==run] +# ['Collected_trigger_rate[Hz]_err']))/1000, # yerr=rate_err[runl], # alpha=0.9, -# ls=' ', marker='o', color=labels[source]['color'], label = labels[source]['source']) -# # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']==run]['Voltage[V]'].values[0])) +# ls=' ', marker='o', color=labels[source]['color'], +# label = labels[source]['source']) +# # label = 'Run {} ({} V)'.format(run, df_mean_rg[df_mean_rg['Run']== +# run]['Voltage[V]'].values[0])) # ax1.legend(frameon=False, prop={'size':10}, # loc="upper left", ncol=1) diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py b/src/nectarchain/trr_test_suite/gui.py similarity index 97% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py rename to src/nectarchain/trr_test_suite/gui.py index 686a45a0..cab2fa32 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -1,14 +1,16 @@ -""" -The `TestRunner` class is a GUI application that allows the user to run various tests and display the results. +"""The `TestRunner` class is a GUI application that allows the user to run various tests +and display the results. The class provides the following functionality: - Allows the user to select a test from a dropdown menu. - Dynamically generates input fields based on the selected test. - Runs the selected test and displays the output in a text box. -- Displays the test results in a plot canvas, with navigation buttons to switch between multiple plots. +- Displays the test results in a plot canvas, with navigation buttons to switch between\ + multiple plots. - Provides a dark-themed UI with custom styling for various UI elements. -The class uses the PyQt5 library for the GUI implementation and the Matplotlib library for plotting the test results. +The class uses the PyQt5 library for the GUI implementation and the Matplotlib library\ + for plotting the test results. """ import argparse @@ -17,11 +19,17 @@ import sys import tempfile +import deadtime +import linearity +import pedestal +import pix_couple_tim_uncertainty +import pix_tim_uncertainty +import trigger_timing from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt5.QtCore import QProcess, QTimer -from PyQt5.QtWidgets import ( +from PyQt6.QtCore import QProcess, QTimer +from PyQt6.QtWidgets import ( QApplication, QComboBox, QGroupBox, @@ -44,12 +52,6 @@ sys.path.append(test_dir) # Import test modules -import deadtime -import linearity -import pedestal -import pix_couple_tim_uncertainty -import pix_tim_uncertainty -import trigger_timing class TestRunner(QWidget): @@ -72,7 +74,8 @@ def __init__(self): self.init_ui() def init_ui(self): - # Main layout: vertical, dividing into two sections (top for controls/plot, bottom for output) + # Main layout: vertical, dividing into two sections (top for controls/plot + # , bottom for output) main_layout = QVBoxLayout() self.setStyleSheet( @@ -116,7 +119,8 @@ def init_ui(self): border-radius: 5px; /* Rounded corners */ } QPushButton:disabled { - background-color: rgba(76, 175, 80, 0.5); /* Transparent green when disabled */ + background-color: rgba(76, 175, 80, 0.5); /* Transparent green when\ + disabled */ color: rgba(255, 255, 255, 0.5); /* Light text when disabled */ } QPushButton:hover { @@ -321,11 +325,13 @@ def update_parameters(self): help_button.setToolTip(param_info["help"]) # # Use lambda to capture the current param's help text - # help_button.clicked.connect(lambda _, p=param_info["help"]: self.show_help(p)) + # help_button.clicked.connect(lambda _, p=param_info["help"]: + # self.show_help(p)) # Add the help button to the layout (next to the label) param_layout.addWidget(help_button) - param_layout.addStretch() # Add stretch to push the help button to the right + param_layout.addStretch() # Add stretch to push the help button to + # the right # Add the horizontal layout (label + help button) to the main layout self.param_layout.addLayout(param_layout) diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py b/src/nectarchain/trr_test_suite/linearity.py similarity index 87% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py rename to src/nectarchain/trr_test_suite/linearity.py index ea62de75..b62461d0 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py +++ b/src/nectarchain/trr_test_suite/linearity.py @@ -8,8 +8,9 @@ import matplotlib.pyplot as plt import numpy as np from lmfit.models import Model -from tools_components import LinearityTestTool -from utils import ( + +from .tools_components import LinearityTestTool +from .utils import ( err_ratio, err_sum, linear_fit_function, @@ -19,8 +20,7 @@ def get_args(): - """ - Parses command-line arguments for the linearity test script. + """Parses command-line arguments for the linearity test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. @@ -28,10 +28,18 @@ def get_args(): parser = argparse.ArgumentParser( description="Linearity test B-TEL-1390 & Intensity resolution B-TEL-1010. \n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween), a corresponding transmission list and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 500) and the number of pixels used (default 70).\n" + + "According to the nectarchain component interface, \ + you have to set a NECTARCAMDATA environment variable\ + in the folder where you have the data from your runs\ + or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween), a\ + corresponding transmission list and an output directory to save the \ + final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed\ + (default 500) and the number of pixels used (default 70).\n" ) parser.add_argument( "-r", @@ -75,19 +83,31 @@ def get_args(): def main(): - """ - The `main()` function is the entry point of the linearity test script. It parses the command-line arguments, processes the specified runs, and generates plots to visualize the linearity and charge resolution of the detector. The function performs the following key steps: - - 1. Parses the command-line arguments using the `get_args()` function, which sets up the argument parser and handles the input parameters. - 2. Iterates through the specified run list, processing each run using the `LinearityTestTool` class. This tool initializes, sets up, starts, and finishes the processing for each run, returning the relevant output data. - 3. Normalizes the high-gain and low-gain charge values using the charge value at 0.01 transmission. + """The `main()` function is the entry point of the linearity test script. It parses + the command-line arguments, processes the specified runs, and generates plots to + visualize the linearity and charge resolution of the detector. The function performs + the following key steps: + + 1. Parses the command-line arguments using the `get_args()` function, which sets up\ + the argument parser and handles the input parameters. + 2. Iterates through the specified run list, processing each run using the\ + `LinearityTestTool` class. This tool initializes, sets up, starts, and finishes\ + the processing for each run, returning the relevant output data. + 3. Normalizes the high-gain and low-gain charge values using the charge value at\ + 0.01 transmission. 4. Generates three subplots: - - The first subplot shows the estimated charge vs. the true charge, with the fitted linear function for both high-gain and low-gain channels. - - The second subplot shows the residuals between the estimated and true charge, as a percentage. - - The third subplot shows the ratio of high-gain to low-gain charge, with a fitted linear function. - 5. Saves the generated plots to the specified output directory, and optionally saves temporary plot files for a GUI. - 6. Generates an additional plot to visualize the charge resolution, including the statistical limit. - 7. Saves the charge resolution plot to the specified output directory, and optionally saves a temporary plot file for a GUI. + - The first subplot shows the estimated charge vs. the true charge, with the fitted\ + linear function for both high-gain and low-gain channels. + - The second subplot shows the residuals between the estimated and true charge, as\ + a percentage. + - The third subplot shows the ratio of high-gain to low-gain charge, with a fitted\ + linear function. + 5. Saves the generated plots to the specified output directory, and optionally\ + saves temporary plot files for a GUI. + 6. Generates an additional plot to visualize the charge resolution, including the\ + statistical limit. + 7. Saves the charge resolution plot to the specified output directory, and\ + optionally saves a temporary plot file for a GUI. """ parser = get_args() args = parser.parse_args() @@ -178,7 +198,7 @@ def main(): axs[2].axvspan(10, 1000, alpha=0.2, color="orange") axs[2].set_xlabel("Illumination charge [p.e.]") - for channel, (channel_charge, channel_std, name) in enumerate( + for _, (channel_charge, channel_std, name) in enumerate( zip( [charge_norm_hg, charge_norm_lg], [std_norm_hg, std_norm_lg], diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py b/src/nectarchain/trr_test_suite/pedestal.py similarity index 82% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py rename to src/nectarchain/trr_test_suite/pedestal.py index b20d2fdb..17185a38 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py +++ b/src/nectarchain/trr_test_suite/pedestal.py @@ -7,13 +7,13 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import PedestalTool -from utils import adc_to_pe, pe2photons + +from .tools_components import PedestalTool +from .utils import adc_to_pe, pe2photons def get_args(): - """ - Parses command-line arguments for the linearity test script. + """Parses command-line arguments for the linearity test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. @@ -21,10 +21,17 @@ def get_args(): parser = argparse.ArgumentParser( description="Pedestal substraction test B-TEL-1370.\n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1200).\n" + + "According to the nectarchain component interface, you have to set\ + a NECTARCAMDATA environment variable in the folder where you have\ + the data from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetwee\ + n) and an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be\ + downloaded through DIRAC.\n For the purposes of testing this script,\ + default data is from the runs used for this test in the\ + TRR document.\n" + + "You can optionally specify the number of events to be processed\ + (default 1200).\n" ) parser.add_argument( "-r", @@ -39,7 +46,8 @@ def get_args(): "-e", "--evts", type=int, - help="Number of events to process from each run. Default is 1200. 4000 or more gives best results but takes some time", + help="Number of events to process from each run. Default is 1200. 4000 or more\ + gives best results but takes some time", required=False, default=10, ) @@ -59,13 +67,15 @@ def get_args(): def main(): - """ - The main function that runs the pedestal subtraction test. It parses command-line arguments, processes the specified runs, and generates two plots: + """The main function that runs the pedestal subtraction test. It parses command-line + arguments, processes the specified runs, and generates two plots: 1. A 2D heatmap of the pedestal RMS for all events and pixels. - 2. A line plot of the mean pedestal RMS for each pixel, with the CTA requirement range highlighted. + 2. A line plot of the mean pedestal RMS for each pixel, with the CTA requirement\ + range highlighted. - The function also saves the generated plots to the specified output directory, and optionally saves the first plot to a temporary output file. + The function also saves the generated plots to the specified output directory,\ + and optionally saves the first plot to a temporary output file. """ parser = get_args() diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py similarity index 71% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py rename to src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py index f57f145f..ad599075 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py @@ -9,25 +9,33 @@ def get_args(): - """ - Parses command-line arguments for the pix_couple_tim_uncertainty_test.py script. + """Parses command-line arguments for the pix_couple_tim_uncertainty_test.py script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( - description="Time resolution (timing uncertainty between couples of pixels) test B-TEL-1030.\n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1000). Takes a lot of time.\n" + description="Time resolution (timing uncertainty between couples of pixels)\ + test B-TEL-1030.\n" + + "According to the nectarchain component interface, you have to set a\ + NECTARCAMDATA\ + environment variable in the folder where you have the data from your runs\ + or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween) and\ + an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed (default\ + 1000). Takes a lot of time.\n" ) parser.add_argument( "-r", "--runlist", type=int, nargs="+", - help="List of runs (numbers separated by space). You can put just one run, default 3292", + help="List of runs (numbers separated by space). You can put just one run,\ + default 3292", required=False, default=[3292], ) @@ -35,7 +43,8 @@ def get_args(): "-e", "--evts", type=int, - help="Number of events to process from each run. Default is 100. 1000 or more gives best results but takes some time", + help="Number of events to process from each run. Default is 100. 1000 or\ + more gives best results but takes some time", required=False, default=100, ) @@ -45,7 +54,9 @@ def get_args(): type=str, help=".csv file with pmt transit time corrections", required=False, - default="../transit_time/hv_pmt_tom_correction_laser_measurement_per_pixel_fit_sqrt_hv_newmethod.csv", + default="../transit_time/\ + hv_pmt_tom_correction_laser_measurement_per_pixel_fit_\ + sqrt_hv_newmethod.csv", ) parser.add_argument( "-o", @@ -63,20 +74,26 @@ def get_args(): def main(): - """ - Generates a plot of the RMS of the time-of-maximum (TOM) difference for pairs of pixels, with a visualization of the CTA requirement. + """Generates a plot of the RMS of the time-of-maximum (TOM) difference for pairs of + pixels, with a visualization of the CTA requirement. - The script processes a list of runs, calculates the TOM difference with and without transit time corrections, and plots the distribution of the RMS of the corrected TOM differences. The CTA requirement of 2 ns RMS is visualized on the plot. + The script processes a list of runs, calculates the TOM difference with and without + transit time corrections, and plots the distribution of the RMS of the corrected TOM + differences. The CTA requirement of 2 ns RMS is visualized on the plot. - The script takes several command-line arguments, including the list of runs to process, the number of events to process per run, the path to a CSV file with PMT transit time corrections, and the output directory for the plot. + The script takes several command-line arguments, including the list of runs to + process, the number of events to process per run, the path to a CSV file with PMT + transit time corrections, and the output directory for the plot. - If a temporary output directory is specified, the plot is also saved to a pickle file in that directory for the gui to use. + If a temporary output directory is specified, the plot is also saved to a pickle + file in that directory for the gui to use. """ parser = get_args() args = parser.parse_args() - tt_path = "/Users/dm277349/nectarchain_data/transit_time/hv_pmt_tom_correction_laser_measurement_per_pixel_fit_sqrt_hv_newmethod.csv" + tt_path = "/Users/dm277349/nectarchain_data/transit_time/\ + hv_pmt_tom_correction_laser_measurement_per_pixel_fit_sqrt_hv_newmethod.csv" runlist = args.runlist nevents = args.evts @@ -153,7 +170,7 @@ def main(): arrowprops=dict(color="C4", alpha=0.7, lw=3, arrowstyle="->"), ) - plt.xlabel("RMS of $\Delta t_{\mathrm{TOM}}$ for pairs of pixels [ns]") + plt.xlabel(r"RMS of $\Delta t_{\mathrm{TOM}}$ for pairs of pixels [ns]") plt.ylabel("Normalized entries") plt.gcf() diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py similarity index 78% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py rename to src/nectarchain/trr_test_suite/pix_tim_uncertainty.py index 2c26575b..d4a6c181 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py @@ -12,18 +12,23 @@ def get_args(): - """ - Parses command-line arguments for the pixel timing uncertainty test script. + """Parses command-line arguments for the pixel timing uncertainty test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( description="Systematic pixel timing uncertainty test B-TEL-1380.\n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1200) and the number of pixels used (default 70).\n" + + "According to the nectarchain component interface, you have to set a\ + NECTARCAMDATA environment variable in the folder where you have the data\ + from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween)\ + and an output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed (default\ + 1200) and the number of pixels used (default 70).\n" ) parser.add_argument( "-r", @@ -42,7 +47,8 @@ def get_args(): required=False, default=100, ) - # parser.add_argument('-p','--pixels', type = int, help='Number of pixels used. Default is 70', required=False, default=70) + # parser.add_argument('-p','--pixels', type = int, help='Number of pixels used. + # Default is 70', required=False, default=70) parser.add_argument( "-o", "--output", @@ -58,12 +64,14 @@ def get_args(): def main(): - """ - Processes the pixel timing uncertainty test data and generates a plot. + """Processes the pixel timing uncertainty test data and generates a plot. - The function processes the data from the specified list of runs, calculates the weighted mean RMS and RMS error, and generates a plot of the results. The plot is saved to the specified output directory. + The function processes the data from the specified list of runs, calculates the + weighted mean RMS and RMS error, and generates a plot of the results. The plot is + saved to the specified output directory. - If a temporary output directory is provided, the plot is also saved to a pickle file in that directory for the gui to use. + If a temporary output directory is provided, the plot is also saved to a pickle file + in that directory for the gui to use. """ parser = get_args() @@ -129,7 +137,8 @@ def main(): rms_no_fit_weighted_err = [] for run in range(len(runlist)): - # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run])/np.sum(weights_mu_pix[run])) + # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run])/ + # np.sum(weights_mu_pix[run])) # rms_mu_weighted_err.append(np.sqrt(1/np.sum(weights_mu_pix[run]))) rms_no_fit_weighted.append( np.nansum(rms_no_fit[run] * weights_no_fit_pix[run]) @@ -227,15 +236,33 @@ def main(): # from test_tools_components import TimingResolutionTestTool # import argparse -# parser = argparse.ArgumentParser(description='Systematic pixel timing uncertainty test B-TEL-1380.\n' -# +'According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n' -# +'You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n' -# +'If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n' -# +'You can optionally specify the number of events to be processed (default 1200) and the number of pixels used (default 70).\n') -# parser.add_argument('-r','--runlist', type = int, nargs='+', help='List of runs (numbers separated by space)', required=False) -# parser.add_argument('-e','--evts', type = int, help='Number of events to process from each run. Default is 1200. 4000 or more gives best results but takes some time', required=False, default=1200) -# #parser.add_argument('-p','--pixels', type = int, help='Number of pixels used. Default is 70', required=False, default=70) -# parser.add_argument('-o','--output', type=str, help='Output directory. If none, plot will be saved in current directory', required=False, default='./') +# parser = argparse.ArgumentParser(description='Sy +# stematic pixel timing uncertainty test B-TEL-1380.\n' +# +'According to +# the nectarchain component interface, you have to set a NECTARCAMDATA environment +# variable in the folder where you have the data from your runs or where you want them +# to be downloaded.\n' +# +'You have to g +# ive a list of runs (run numbers with spaces inbetween) and an output directory to save +# the final plot.\n' +# +'If the data i +# s not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes +# of testing this script, default data is from the runs used for this test in the TRR +# document.\n' +# +'You can optio +# nally specify the number of events to be processed (default 1200) and the +# number of +# pixels used (default 70).\n') +# parser.add_argument('-r','--runlist', type = int +# , nargs='+', help='List of runs (numbers separated by space)', required=False) +# parser.add_argument('-e','--evts', type = int, h +# elp='Number of events to process from each run. Default is 1200. 4000 or more gives +# best results but takes some time', required=False, default=1200) +# #parser.add_argument('-p','--pixels', type = int +# , help='Number of pixels used. Default is 70', required=False, default=70) +# parser.add_argument('-o','--output', type=str, h +# elp='Output directory. If none, plot will be saved in current directory', +# required=False, default='./') # args = parser.parse_args() @@ -257,7 +284,8 @@ def main(): # for run in runlist: # print("PROCESSING RUN {}".format(run)) # tool = TimingResolutionTestTool( -# progress_bar=True, run_number=run, max_events=nevents, log_level=20, window_width=16, overwrite=True +# progress_bar=True, run_number=run, max_events=nevents, log_level=20, +# window_width=16, overwrite=True # ) # tool.initialize() # tool.setup() @@ -301,9 +329,11 @@ def main(): # rms_no_fit_weighted_err = [] # for run in range(len(runlist)): -# # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run])/np.sum(weights_mu_pix[run])) +# # rms_mu_weighted.append(np.sum(rms_mu[run]*weights_mu_pix[run]) +# /np.sum(weights_mu_pix[run])) # # rms_mu_weighted_err.append(np.sqrt(1/np.sum(weights_mu_pix[run]))) -# rms_no_fit_weighted.append(np.nansum(rms_no_fit[run]*weights_no_fit_pix[run])/np.nansum(weights_no_fit_pix[run])) +# rms_no_fit_weighted.append(np.nansum(rms_no_fit[run]*weights_no_fit_pix[run]) +# /np.nansum(weights_no_fit_pix[run])) # rms_no_fit_weighted_err.append(np.sqrt(1/np.nansum(weights_no_fit_pix[run]))) @@ -325,7 +355,8 @@ def main(): # plt.axhline(1, ls='--', color='C4', alpha=0.6) -# plt.axhline(1/np.sqrt(12), ls='--', color='gray', alpha=0.7, label='Quantification rms noise') +# plt.axhline(1/np.sqrt(12), ls='--', color='gray', alpha=0.7, label= +# 'Quantification rms noise') # plt.axvspan(20, 1000, alpha=0.1, color='C4') diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py b/src/nectarchain/trr_test_suite/tools_components.py similarity index 83% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py rename to src/nectarchain/trr_test_suite/tools_components.py index a1a4c52e..158ce4c6 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py +++ b/src/nectarchain/trr_test_suite/tools_components.py @@ -21,12 +21,16 @@ # overriding so we can have maxevents in the path def _init_output_path(self): - """ - Initializes the output path for the NectarCAMCalibrationTool. + """Initializes the output path for the NectarCAMCalibrationTool. - If `max_events` is `None`, the output file name will be in the format `{self.name}_run{self.run_number}.h5`. Otherwise, the file name will be in the format `{self.name}_run{self.run_number}_maxevents{self.max_events}.h5`. + If `max_events` is `None`, the output file name will be in the format\ + `{self.name}_run{self.run_number}.h5`. Otherwise, the file name will\ + be in the format\ + `{self.name}_run{self.run_number}_maxevents{self.max_events}.h5`. - The output path is constructed by joining the `NECTARCAMDATA` environment variable (or `/tmp` if not set) with the `tests` subdirectory and the generated file name. + The output path is constructed by joining the `NECTARCAMDATA` environment variable\ + (or `/tmp` if not set) with the `tests` subdirectory and the generated\ + file name. """ if self.max_events is None: @@ -42,8 +46,8 @@ def _init_output_path(self): class ChargeContainer(NectarCAMContainer): - """ - This class contains fields that store various properties and data related to NectarCAM events, including: + """This class contains fields that store various properties and data related to + NectarCAM events, including: - `run_number`: The run number associated with the waveforms. - `npixels`: The number of effective pixels. @@ -81,15 +85,22 @@ class ChargeContainer(NectarCAMContainer): class ChargeComp(NectarCAMComponent): - """ - This class `ChargeComp` is a NectarCAMComponent that processes NectarCAM event data. It extracts the charge information from the waveforms of each event, handling cases of saturated or noisy events. The class has the following configurable parameters: + """This class `ChargeComp` is a NectarCAMComponent that processes NectarCAM event + data. It extracts the charge information from the waveforms of each event, handling + cases of saturated or noisy events. The class has the following configurable + parameters: - `window_shift`: The time in ns before the peak to extract the charge. - `window_width`: The duration of the charge extraction window in ns. - The `__init__` method initializes important members of the component, such as timestamps, event type, event ids, pedestal and charge for both gain channels. - The `__call__` method is the main processing logic, which is called for each event. It extracts the charge information for both high gain and low gain channels, handling various cases such as saturated events and events with no signal. - The `finish` method collects all the processed data and returns a `ChargeContainer` object containing the run number, number of pixels, pixel IDs, UCTS timestamps, event types, event IDs, and the high and low gain charge values. + The `__init__` method initializes important members of the component, such as\ + timestamps, event type, event ids, pedestal and charge for both gain channels. + The `__call__` method is the main processing logic, which is called for each event.\ + It extracts the charge information for both high gain and low gain channels,\ + handling various cases such as saturated events and events with no signal. + The `finish` method collects all the processed data and returns a `ChargeContainer`\ + object containing the run number, number of pixels, pixel IDs, UCTS timestamps,\ + event types, event IDs, and the high and low gain charge values. """ window_shift = Integer( @@ -106,7 +117,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain + # interesting quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -118,7 +130,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__charge_hg = [] self.__charge_lg = [] - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__event_id.append(np.uint32(event.index.event_id)) @@ -131,7 +143,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): wfs.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id]) wfs.append(event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id]) - #####THE JOB IS HERE###### + # ###THE JOB IS HERE#### for i, (pedestal, charge) in enumerate( zip( [self.__pedestal_hg, self.__pedestal_lg], @@ -181,9 +193,11 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): integral[pix] = 0 else: - # x = np.linspace(0,signal_stop[pix]-signal_start[pix],signal_stop[pix]-signal_start[pix]) + # x = np.linspace(0,signal_stop[pix]-signal_start[pix], + # signal_stop[pix]-signal_start[pix]) # spl = UnivariateSpline(x,y) - # integral[pix] = spl.integral(0,signal_stop[pix]-signal_start[pix]) + # integral[pix] = spl.integral(0,signal_stop[pix]- + # signal_start[pix]) integral[pix] = np.sum( wf[pix, signal_start[pix] : signal_stop[pix]] @@ -193,7 +207,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): charge.append(chg) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = ChargeContainer( run_number=ChargeContainer.fields["run_number"].type(self._run_number), @@ -218,10 +232,16 @@ def finish(self): class LinearityTestTool(EventsLoopNectarCAMCalibrationTool): - """ - This class, `LinearityTestTool`, is a subclass of `EventsLoopNectarCAMCalibrationTool`. It is responsible for performing a linearity test on NectarCAM data. The class has a `componentsList` attribute that specifies the list of NectarCAM components to be applied. - - The `finish` method is the main functionality of this class. It reads the charge data from the output file, calculates the mean charge, standard deviation, and standard error for both the high gain and low gain channels, and returns these values. This information can be used to assess the linearity of the NectarCAM system. + """This class, `LinearityTestTool`, is a subclass of + `EventsLoopNectarCAMCalibrationTool`. It is responsible for performing a linearity + test on NectarCAM data. The class has a `componentsList` attribute that specifies + the list of NectarCAM components to be applied. + + The `finish` method is the main functionality of this class. It reads the charge\ + data from the output file, calculates the mean charge, standard deviation,\ + and standard error for both the high gain and low gain channels, and\ + returns these values. This information can be used to assess\ + the linearity of the NectarCAM system. """ name = "LinearityTestTool" @@ -255,7 +275,7 @@ def finish(self, *args, **kwargs): charge_hg.extend(tup[6]) charge_lg.extend(tup[7]) - except: + except Exception: break output_file.close() @@ -288,7 +308,8 @@ class ToMContainer(NectarCAMContainer): event_type (np.ndarray[np.uint8]): The trigger event types. event_id (np.ndarray[np.uint32]): The event IDs. charge_hg (np.ndarray[np.float64]): The mean high gain charge per event. - tom_no_fit (np.ndarray[np.float64]): The time of maximum from the data (no fitting). + tom_no_fit (np.ndarray[np.float64]): The time of maximum from\ + the data (no fitting). good_evts (np.ndarray[np.uint32]): The IDs of the good (non-cosmic ray) events. """ @@ -317,11 +338,13 @@ class ToMContainer(NectarCAMContainer): ) # tom_mu = Field( - # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of maximum of signal fitted with gaussian" + # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of maximum of + # signal fitted with gaussian" # ) # tom_sigma = Field( - # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of fitted maximum sigma" + # type=np.ndarray, dtype=np.float64, ndim=2, description="Time of fitted + # maximum sigma" # ) tom_no_fit = Field( type=np.ndarray, @@ -338,14 +361,25 @@ class ToMContainer(NectarCAMContainer): class ToMComp(NectarCAMComponent): - """ - This class, `ToMComp`, is a component of the NectarCAM system that is responsible for processing waveform data. It has several configurable parameters, including the width and shift before the peak of the time window for charge extraction, the peak height threshold. - - The `__init__` method initializes some important component members, such as timestamps, event type, event ids, pedestal and charge values for both gain channels. - - The `__call__` method is the main entry point for processing an event. It extracts the waveform data, calculates the pedestal, charge, and time of maximum (ToM) for each pixel, and filters out events that do not meet the peak height threshold. The results are stored in various member variables, which are then returned in the `finish` method. - - The `finish` method collects the processed data from the member variables and returns a `ToMContainer` object, which contains the run number, number of pixels, pixel IDs, UCTS timestamps, event types, event IDs, high-gain charge, ToM without fitting, and IDs of good (non-cosmic ray) events. + """This class, `ToMComp`, is a component of the NectarCAM system that is responsible + for processing waveform data. It has several configurable parameters, including the + width and shift before the peak of the time window for charge extraction, the peak + height threshold. + + The `__init__` method initializes some important component members, such as\ + timestamps, event type, event ids, pedestal and charge values for both gain\ + channels. + + The `__call__` method is the main entry point for processing an event. It extracts\ + the waveform data, calculates the pedestal, charge, and time of maximum (ToM)\ + for each pixel, and filters out events that do not meet the peak\ + height threshold. The results are stored in various member variables,\ + which are then returned in the `finish` method. + + The `finish` method collects the processed data from the member variables and\ + returns a `ToMContainer` object, which contains the run number, number of\ + pixels, pixel IDs, UCTS timestamps, event types, event IDs, high-gain\ + charge, ToM without fitting, and IDs of good (non-cosmic ray) events. """ window_shift = Integer( @@ -367,7 +401,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain + # interesting quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -386,7 +421,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__ff_event_ind = -1 - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__event_id.append(np.uint32(event.index.event_id)) @@ -398,7 +433,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__ff_event_ind += 1 - #####THE JOB IS HERE###### + # #####THE JOB IS HERE###### for i, (pedestal, charge, tom_no_fit) in enumerate( zip([self.__pedestal_hg], [self.__charge_hg], [self.__tom_no_fit]) @@ -483,7 +518,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # # fit # model = Model(gaus) - # params = model.make_params(a=yi[peaks[max_peak_index]] * 3, mu=mean, sigma=sigma) + # params = model.make_params(a=yi[peaks[max_peak_index]] * 3, + # mu=mean, sigma=sigma) # result = model.fit(y_fit, params, x=x_fit) # result_sigma = result.params['sigma'].value @@ -510,12 +546,16 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # change_grad_pos_left = 3 # change_grad_pos_right = 3 # mean = xi[peaks[max_peak_index]] - # sigma = change_grad_pos_right + change_grad_pos_left # define window for the gaussian fit + # sigma = change_grad_pos_right + change_grad_pos_left # define + # window for the gaussian fit - # x_fit = xi[peaks[max_peak_index]-change_grad_pos_left:peaks[max_peak_index]+change_grad_pos_right] - # y_fit = yi[peaks[max_peak_index]-change_grad_pos_left:peaks[max_peak_index]+change_grad_pos_right] + # x_fit = xi[peaks[max_peak_index]-change_grad_pos_left:peaks + # [max_peak_index]+change_grad_pos_right] + # y_fit = yi[peaks[max_peak_index]-change_grad_pos_left:peaks + # [max_peak_index]+change_grad_pos_right] # model = Model(gaus) - # params = model.make_params(a=yi[peaks[max_peak_index]],mu=mean,sigma=sigma) + # params = model.make_params(a=yi[peaks[max_peak_index]], + # mu=mean,sigma=sigma) # result = model.fit(y_fit, params, x=x_fit) max_position_x_prefit = xi[peaks[max_peak_index]] @@ -523,7 +563,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # result_mu = result.params['mu'].value else: - # index_x_window_min = list(xi).index(closest_value(xi, signal_start[pix])) + # index_x_window_min = list(xi).index(closest_value(xi, + # signal_start[pix])) charge_sum = y[ signal_start[pix] : signal_start[pix] + self.window_width ].sum() @@ -533,10 +574,12 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # result_mu = -1 else: - # If no maximum is found, the integration is done between 20 and 36 ns. + # If no maximum is found, the integration is done between 20 and 36 + # ns. signal_start[pix] = 20 - # index_x_window_min = list(xi).index(closest_value(xi, signal_start[pix])) + # index_x_window_min = list(xi).index(closest_value(xi, + # signal_start[pix])) charge_sum = y[ signal_start[pix] : signal_start[pix] + self.window_width ].sum() @@ -562,7 +605,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): # print("is good evt") self.__good_evts.append(self.__ff_event_ind) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = ToMContainer( run_number=ToMContainer.fields["run_number"].type(self._run_number), @@ -587,12 +630,20 @@ def finish(self): class TimingResolutionTestTool(EventsLoopNectarCAMCalibrationTool): - """ - This class, `TimingResolutionTestTool`, is a subclass of `EventsLoopNectarCAMCalibrationTool` and is used to perform timing resolution tests on NectarCAM data. It reads the output data from the `ToMContainer` dataset and processes the charge, timing, and event information to calculate the timing resolution and mean charge in photoelectrons. - - The `finish()` method is the main entry point for this tool. It reads the output data from the HDF5 file, filters the data to remove cosmic ray events, and then calculates the timing resolution and mean charge per photoelectron. The timing resolution is calculated using a weighted mean and variance approach, with an option to use a bootstrapping method to estimate the error on the RMS value. - - The method returns the RMS of the timing resolution, the error on the RMS, and the mean charge in photoelectrons. + """This class, `TimingResolutionTestTool`, is a subclass of + `EventsLoopNectarCAMCalibrationTool` and is used to perform timing resolution tests + on NectarCAM data. It reads the output data from the `ToMContainer` dataset and + processes the charge, timing, and event information to calculate the timing + resolution and mean charge in photoelectrons. + + The `finish()` method is the main entry point for this tool. It reads the output + data from the HDF5 file, filters the data to remove cosmic ray events, and then + calculates the timing resolution and mean charge per photoelectron. The timing + resolution is calculated using a weighted mean and variance approach, with an option + to use a bootstrapping method to estimate the error on the RMS value. + + The method returns the RMS of the timing resolution, the error on the RMS, and the + mean charge in photoelectrons. """ name = "TimingResolutionTestTool" @@ -625,7 +676,7 @@ def finish(self, bootstrap=False, *args, **kwargs): charge_all.extend(tup[6]) tom_no_fit_all.extend(tup[7]) good_evts.extend(tup[8]) - except: + except Exception: break output_file.close() @@ -640,9 +691,11 @@ def finish(self, bootstrap=False, *args, **kwargs): # print(good_evts) charge = charge_all[good_evts] mean_charge_pe = np.mean(np.mean(charge, axis=0)) / 58.0 - # tom_mu = np.array(tom_mu_all[good_evts]).reshape(len(good_evts),output[0].npixels) + # tom_mu = np.array(tom_mu_all[good_evts]).reshape(len(good_evts), + # output[0].npixels) - # tom_sigma = np.array(tom_sigma_all[good_evts]).reshape(len(good_evts),output[0].npixels) + # tom_sigma = np.array(tom_sigma_all[good_evts]).reshape(len(good_evts), + # output[0].npixels) tom_no_fit = np.array(tom_no_fit_all[good_evts]).reshape( len(good_evts), npixels ) @@ -679,7 +732,8 @@ def finish(self, bootstrap=False, *args, **kwargs): bootsample = np.random.choice( sample, size=int(3 / 4 * (len(sample))), replace=True ) - # print(len(bootsample), bootsample.mean(), bootsample.std()) + # print(len(bootsample), bootsample.mean(), + # bootsample.std()) boot_rms.append(bootsample.std()) # simulated mean of rms bootrms_mean = np.mean(boot_rms) @@ -707,14 +761,15 @@ def finish(self, bootstrap=False, *args, **kwargs): rms[pix] = np.sqrt(weighted_variance) # print("RMS:", rms[pix]) - # Compute the total number of data points (sum of histogram values, i.e. N) + # Compute the total number of data points (sum of histogram + # values, i.e. N) N = np.sum(hist_values) # print("Total number of events (N):", N) # Error on the standard deviation err[pix] = rms[pix] / np.sqrt(2 * N) # print("Error on RMS:", err[pix]) - except: + except Exception: # no data rms[pix] = np.nan err[pix] = np.nan @@ -723,17 +778,22 @@ def finish(self, bootstrap=False, *args, **kwargs): class ToMPairsTool(EventsLoopNectarCAMCalibrationTool): - """ - This class, `ToMPairsTool`, is an `EventsLoopNectarCAMCalibrationTool` that is used to process ToM (Time of maximum) data from NectarCAM. + """This class, `ToMPairsTool`, is an `EventsLoopNectarCAMCalibrationTool`\ + that is used to process ToM (Time of maximum) data from NectarCAM. The `finish` method has the following functionalities: - - It reads in ToM data from an HDF5 file and applies a transit time correction to the ToM values using a provided lookup table. - - It calculates the time difference between ToM pairs for both corrected and uncorrected ToM values. - - It returns the uncorrected ToM values, the corrected ToM values, the pixel IDs, and the time difference calculations for the uncorrected and corrected ToM values. - - The class has several configurable parameters, including the list of NectarCAM components to apply, the maximum number of events to process, and the output file path. - + - It reads in ToM data from an HDF5 file and applies a transit time correction to\ + the ToM values using a provided lookup table. + - It calculates the time difference between ToM pairs for both corrected and\ + uncorrected ToM values. + - It returns the uncorrected ToM values, the corrected ToM values, the pixel IDs,\ + and the time difference calculations for the uncorrected and corrected\ + ToM values. + + The class has several configurable parameters, including the list of NectarCAM\ + components to apply, the maximum number of events to process, and the output\ + file path. """ name = "ToMPairsTool" @@ -765,7 +825,7 @@ def finish(self, *args, **kwargs): try: pixels_id.extend(tup[2]) tom_no_fit_all.extend(tup[7]) - except: + except Exception: break output_file.close() @@ -851,8 +911,8 @@ def finish(self, *args, **kwargs): class PedestalContainer(NectarCAMContainer): - """ - Attributes of the PedestalContainer class that store various data related to the pedestal of a NectarCAM event. + """Attributes of the PedestalContainer class that store various data related to the + pedestal of a NectarCAM event. Attributes: run_number (np.uint16): The run number associated with the waveforms. @@ -914,21 +974,30 @@ class PedestalContainer(NectarCAMContainer): class PedestalComp(NectarCAMComponent): - """ - The `PedestalComp` class is a NectarCAMComponent that is responsible for processing the pedestal and RMS of the high and low gain waveforms for each event. - - The `__init__` method initializes the `PedestalComp` class. It sets up several member variables to store pedestal related data such as timestamps, event types, event IDs, pedestal and pedestal rms values for both gains. - - The `__call__` method is called for each event, and it processes the waveforms to calculate the pedestal and RMS for the high and low gain channels. The results are stored in the class attributes `__pedestal_hg`, `__pedestal_lg`, `__rms_ped_hg`, and `__rms_ped_lg`. - - The `finish` method is called at the end of processing, and it returns a `PedestalContainer` object containing the calculated pedestal and RMS values, as well as other event information. + """The `PedestalComp` class is a NectarCAMComponent that is responsible for + processing the pedestal and RMS of the high and low gain waveforms for each event. + + The `__init__` method initializes the `PedestalComp` class. It sets up several\ + member variables to store pedestal related data such as timestamps,\ + event types,\ + event IDs, pedestal and pedestal rms values for both gains. + + The `__call__` method is called for each event, and it processes the waveforms to\ + calculate the pedestal and RMS for the high and low gain channels. The results\ + are stored in the class attributes `__pedestal_hg`, `__pedestal_lg`, \ + `__rms_ped_hg`, and `__rms_ped_lg`. + + The `finish` method is called at the end of processing, and it returns a\ + `PedestalContainer` object containing the calculated pedestal and RMS values\ + , as well as other event information. """ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain interesting + # quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -941,7 +1010,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__rms_ped_hg = [] self.__rms_ped_lg = [] - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__event_id.append(np.uint32(event.index.event_id)) @@ -954,7 +1023,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): wfs.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id]) wfs.append(event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id]) - #####THE JOB IS HERE###### + # #####THE JOB IS HERE###### for i, (pedestal, rms_pedestal) in enumerate( zip( @@ -972,7 +1041,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): pedestal.append(ped) rms_pedestal.append(rms_ped) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = PedestalContainer( run_number=PedestalContainer.fields["run_number"].type(self._run_number), @@ -1002,12 +1071,16 @@ def finish(self): class PedestalTool(EventsLoopNectarCAMCalibrationTool): - """ - This class is a part of the PedestalTool, which is an EventsLoopNectarCAMCalibrationTool. + """This class is a part of the PedestalTool, which is an + EventsLoopNectarCAMCalibrationTool. - The finish() method opens the output file, which is an HDF5 file, and extracts the `rms_ped_hg` (root mean square of the high gain pedestal) values from the `PedestalContainer` dataset. Finally, it closes the output file and returns the list of `rms_ped_hg` values. + The finish() method opens the output file, which is an HDF5 file,\ + and extracts the `rms_ped_hg` (root mean square of the high gain pedestal)\ + values from the `PedestalContainer` dataset. Finally, it closes the output\ + file and returns the list of `rms_ped_hg` values. - This method is used to post-process the output of the PedestalTool and extract specific information from the generated HDF5 file. + This method is used to post-process the output of the PedestalTool and extract\ + specific information from the generated HDF5 file. """ name = "PedestalTool" @@ -1042,7 +1115,7 @@ def finish(self, *args, **kwargs): for tup in data: try: rms_ped_hg.extend(tup[8]) - except: + except Exception: break output_file.close() @@ -1051,8 +1124,8 @@ def finish(self, *args, **kwargs): class UCTSContainer(NectarCAMContainer): - """ - Defines the fields for the UCTSContainer class, which is used to store various data related to UCTS events. + """Defines the fields for the UCTSContainer class, which is used to store various + data related to UCTS events. The fields include: - `run_number`: The run number associated with the waveforms. @@ -1096,12 +1169,15 @@ class UCTSContainer(NectarCAMContainer): class UCTSComp(NectarCAMComponent): - """ - The `__init__` method initializes the `UCTSComp` class, which is a NectarCAMComponent. It sets up several member variables to store UCTS related data, such as timestamps, event types, event IDs, busy counters, and event counters. + """The `__init__` method initializes the `UCTSComp` class, which is a + NectarCAMComponent. It sets up several member variables to store UCTS related data, + such as timestamps, event types, event IDs, busy counters, and event counters. - The `__call__` method is called for each event, and it appends the UCTS-related data from the event to the corresponding member variables. + The `__call__` method is called for each event, and it appends the UCTS-related\ + data from the event to the corresponding member variables. - The `finish` method creates and returns a `UCTSContainer` object, which is a container for the UCTS-related data that was collected during the event loop. + The `finish` method creates and returns a `UCTSContainer` object, which is a\ + container for the UCTS-related data that was collected during the event loop. """ window_shift = Integer( @@ -1120,7 +1196,8 @@ def __init__( super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process + # If you want you can add here members of MyComp, they will contain interesting + # quantity during the event loop process self.__ucts_timestamp = [] self.__event_type = [] @@ -1130,7 +1207,7 @@ def __init__( self.excl_muons = None self.__mean_event_charge = [] - ##This method need to be defined ! + # This method need to be defined ! def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): take_event = True @@ -1193,7 +1270,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): if self.excl_muons: self.__mean_event_charge.append(mean_charge) - ##This method need to be defined ! + # This method need to be defined ! def finish(self): output = UCTSContainer( run_number=UCTSContainer.fields["run_number"].type(self._run_number), @@ -1218,10 +1295,16 @@ def finish(self): class DeadtimeTestTool(EventsLoopNectarCAMCalibrationTool): - """ - The `DeadtimeTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that is used to test the deadtime of NectarCAM. - - The `finish` method is responsible for reading the data from the HDF5 file, extracting the relevant information (UCTS timestamps, event counters, and busy counters), and calculating the deadtime-related metrics. The method returns the UCTS timestamps, the time differences between consecutive UCTS timestamps, the event counters, the busy counters, the collected trigger rate, the total time, and the deadtime percentage. + """The `DeadtimeTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that is + used to test the deadtime of NectarCAM. + + The `finish` method is responsible for reading the data from the HDF5 file,\ + extracting the relevant information (UCTS timestamps, event counters, and\ + busy counters), and calculating the deadtime-related metrics. The method\ + returns the UCTS timestamps, the time differences between consecutive\ + UCTS timestamps, the event counters, the busy counters,\ + the collected\ + trigger rate, the total time, and the deadtime percentage. """ name = "DeadtimeTestTool" @@ -1250,7 +1333,7 @@ def finish(self, *args, **kwargs): ucts_timestamps.extend(tup[3]) event_counter.extend(tup[7]) busy_counter.extend(tup[6]) - except: + except Exception: break # print(output_file.keys()) # tom_mu_all= output[0].tom_mu @@ -1285,10 +1368,15 @@ def finish(self, *args, **kwargs): class TriggerTimingTestTool(EventsLoopNectarCAMCalibrationTool): - """ - The `TriggerTimingTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that is used to test the trigger timing of NectarCAM. - - The `finish` method is responsible for reading the data from the HDF5 file, extracting the relevant information (UCTS timestamps), and calculating the RMS value of the difference between consecutive triggers. The method returns the UCTS timestamps, the time differences between consecutive triggers for events concerning more than 10 pixels (non-muon related events). + """The `TriggerTimingTestTool` class is an `EventsLoopNectarCAMCalibrationTool` that + is used to test the trigger timing of NectarCAM. + + The `finish` method is responsible for reading the data from the HDF5 file,\ + extracting the relevant information (UCTS timestamps), and calculating\ + the RMS value of the difference between consecutive triggers. The method\ + returns the UCTS timestamps, the time differences between consecutive\ + triggers for events concerning more than 10 pixels (non-muon\ + related events). """ name = "TriggerTimingTestTool" @@ -1323,7 +1411,7 @@ def finish(self, *args, **kwargs): ucts_timestamps.extend(tup[3]) charge_per_event.extend(tup[4]) - except: + except Exception: break # print(output_file.keys()) # tom_mu_all= output[0].tom_mu diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py b/src/nectarchain/trr_test_suite/trigger_timing.py similarity index 81% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py rename to src/nectarchain/trr_test_suite/trigger_timing.py index 6dcc8f7d..0d679959 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py +++ b/src/nectarchain/trr_test_suite/trigger_timing.py @@ -12,18 +12,23 @@ def get_args(): - """ - Parses command-line arguments for the deadtime test script. + """Parses command-line arguments for the deadtime test script. Returns: argparse.ArgumentParser: The parsed command-line arguments. """ parser = argparse.ArgumentParser( description="Trigger Timing Test B-TEL-1410. \n" - + "According to the nectarchain component interface, you have to set a NECTARCAMDATA environment variable in the folder where you have the data from your runs or where you want them to be downloaded.\n" - + "You have to give a list of runs (run numbers with spaces inbetween) and an output directory to save the final plot.\n" - + "If the data is not in NECTARCAMDATA, the files will be downloaded through DIRAC.\n For the purposes of testing this script, default data is from the runs used for this test in the TRR document.\n" - + "You can optionally specify the number of events to be processed (default 1000).\n" + + "According to the nectarchain component interface, you have to set\ + a NECTARCAMDATA environment variable in the folder where you have the data\ + from your runs or where you want them to be downloaded.\n" + + "You have to give a list of runs (run numbers with spaces inbetween) and an\ + output directory to save the final plot.\n" + + "If the data is not in NECTARCAMDATA, the files will be downloaded through\ + DIRAC.\n For the purposes of testing this script, default data is from the\ + runs used for this test in the TRR document.\n" + + "You can optionally specify the number of events to be processed\ + (default 1000).\n" ) parser.add_argument( "-r", @@ -58,15 +63,21 @@ def get_args(): def main(): - """ - Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and B-TEL-1270. + """Runs the deadtime test script, which performs deadtime tests B-TEL-1260 and + B-TEL-1270. - The script takes command-line arguments to specify the list of runs, corresponding sources, number of events to process, and output directory. It then processes the data for each run, performs an exponential fit to the deadtime distribution, and generates two plots: + The script takes command-line arguments to specify the list of runs, corresponding\ + sources, number of events to process, and output directory. It then processes\ + the data for each run, performs an exponential fit to the deadtime\ + distribution, and generates two plots: - 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA requirement indicated. - 2. A plot of the rate from the fit vs. the collected trigger rate, with the relative difference shown in the bottom panel. + 1. A plot of deadtime percentage vs. collected trigger rate, with the CTA\ + requirement indicated. + 2. A plot of the rate from the fit vs. the collected trigger rate, with the\ + relative difference shown in the bottom panel. - The script also saves the generated plots to the specified output directory, and optionally saves them to a temporary output directory for use in a GUI. + The script also saves the generated plots to the specified output directory,\ + and optionally saves them to a temporary output directory for use in a GUI. """ parser = get_args() @@ -141,7 +152,8 @@ def main(): cta_requirement_y = 5 # Y-value for the CTA requirement ax.axhline(y=cta_requirement_y, color="purple", linestyle="--") - # Add the small vertical arrows starting from the CTA requirement line and pointing downwards + # Add the small vertical arrows starting from the CTA requirement line and pointing + # downwards arrow_positions = [20, 80, 200] # X-positions for the arrows for x_pos in arrow_positions: ax.annotate( @@ -151,7 +163,8 @@ def main(): arrowprops=dict(arrowstyle="->", color="purple", lw=1.5), ) # Arrow pointing downwards - # Add the CTA requirement label exactly above the dashed line, centered between arrows + # Add the CTA requirement label exactly above the dashed line, centered between + # arrows ax.text( 140, cta_requirement_y + 0.5, diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py b/src/nectarchain/trr_test_suite/utils.py similarity index 88% rename from src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py rename to src/nectarchain/trr_test_suite/utils.py index 27f8b202..b8fec4c9 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py +++ b/src/nectarchain/trr_test_suite/utils.py @@ -241,13 +241,15 @@ def pe_from_intensity_percentage( percent_from_calibration=intensity_percent, known_charge=intensity_to_charge, ): - """ - Converts a percentage of intensity to the corresponding charge value based on a known calibration. + """Converts a percentage of intensity to the corresponding charge value based on a + known calibration. Args: percent (numpy.ndarray): The percentage of intensity to convert to charge. - percent_from_calibration (numpy.ndarray, optional): The known percentages of intensity used in the calibration. Defaults to `intensity_percent`. - known_charge (numpy.ndarray, optional): The known charge values corresponding to the calibration percentages. Defaults to `intensity_to_charge`. + percent_from_calibration (numpy.ndarray, optional): The known percentages of\ + intensity used in the calibration. Defaults to `intensity_percent`. + known_charge (numpy.ndarray, optional): The known charge values corresponding\ + to the calibration percentages. Defaults to `intensity_to_charge`. Returns: numpy.ndarray: The charge values corresponding to the input percentages. @@ -265,8 +267,7 @@ def pe_from_intensity_percentage( # functions by federica def linear_fit_function(x, a, b): - """ - Computes a linear function of the form `a*x + b`. + """Computes a linear function of the form `a*x + b`. Args: x (float): The input value. @@ -280,8 +281,7 @@ def linear_fit_function(x, a, b): def second_degree_fit_function(x, a, b, c): - """ - Computes a quadratic function of the form `a*(x**2) + b*x + c`. + """Computes a quadratic function of the form `a*(x**2) + b*x + c`. Args: x (float): The input value. @@ -296,8 +296,7 @@ def second_degree_fit_function(x, a, b, c): def third_degree_fit_function(x, a, b, c, d): - """ - Computes a function of the form `(a*x + b)/(1+c) + d`. + """Computes a function of the form `(a*x + b)/(1+c) + d`. Args: x (float): The input value. @@ -313,8 +312,7 @@ def third_degree_fit_function(x, a, b, c, d): def fit_function_hv(x, a, b): - """ - Computes a function of the form `a/sqrt(x) + b`. + """Computes a function of the form `a/sqrt(x) + b`. Args: x (float): The input value. @@ -328,15 +326,16 @@ def fit_function_hv(x, a, b): def err_ratio(nominator, denominator, err_norm, err_denom, cov_nom_den=0): - """ - Computes the error ratio for a given nominator, denominator, and their respective errors. + """Computes the error ratio for a given nominator, denominator, and their respective + errors. Args: nominator (float): The nominator value. denominator (float): The denominator value. err_norm (float): The error of the nominator. err_denom (float): The error of the denominator. - cov_nom_den (float, optional): The covariance between the nominator and denominator. Defaults to 0. + cov_nom_den (float, optional): The covariance between the nominator and\ + denominator. Defaults to 0. Returns: float: The error ratio. @@ -351,10 +350,11 @@ def err_ratio(nominator, denominator, err_norm, err_denom, cov_nom_den=0): def err_sum(err_a, err_b, cov_a_b=0): - """ - Computes the square root of the sum of the squares of `err_a` and `err_b`, plus twice the covariance `cov_a_b`. + """Computes the square root of the sum of the squares of `err_a` and `err_b`, plus + twice the covariance `cov_a_b`. - This function is used to calculate the combined error of two values, taking into account their individual errors and the covariance between them. + This function is used to calculate the combined error of two values, taking into\ + account their individual errors and the covariance between them. Args: err_a (float): The error of the first value. @@ -369,18 +369,22 @@ def err_sum(err_a, err_b, cov_a_b=0): # from stackoverflow def argmedian(x, axis=None): - """ - Returns the index of the median element in the input array `x` along the specified axis. + """Returns the index of the median element in the input array `x` along the + specified axis. - If `axis` is `None`, the function returns the index of the median element in the flattened array. - Otherwise, it computes the argmedian along the specified axis and returns an array of indices. + If `axis` is `None`, the function returns the index of the median element in\ + the flattened array. + Otherwise, it computes the argmedian along the specified axis and returns an\ + array of indices. Args: x (numpy.ndarray): The input array. - axis (int or None, optional): The axis along which to compute the argmedian. If `None`, the argmedian is computed on the flattened array. + axis (int or None, optional): The axis along which to compute the argmedian.\ + If `None`, the argmedian is computed on the flattened array. Returns: - int or numpy.ndarray: The index or indices of the median element(s) in the input array. + int or numpy.ndarray: The index or indices of the median element(s) in the\ + input array. """ if axis is None: return np.argpartition(x, len(x) // 2)[len(x) // 2] @@ -392,8 +396,8 @@ def argmedian(x, axis=None): def pe2photons(x): - """ - Converts the input value `x` from photons to photoelectrons (PE) by multiplying it by 4. + """Converts the input value `x` from photons to photoelectrons (PE) by multiplying + it by 4. Args: x (float): The input value in photons. @@ -405,8 +409,8 @@ def pe2photons(x): def photons2pe(x): - """ - Converts the input value `x` from photoelectrons (PE) to photons by dividing it by 4. + """Converts the input value `x` from photoelectrons (PE) to photons by dividing it + by 4. Args: x (float): The input value in photoelectrons. @@ -419,8 +423,8 @@ def photons2pe(x): # from federica's notebook class ExponentialFitter: - """ - Represents an exponential fitter class that computes the expected distribution and the minus 2 log likelihood for a given dataset and exponential parameters. + """Represents an exponential fitter class that computes the expected distribution + and the minus 2 log likelihood for a given dataset and exponential parameters. Attributes: datas (numpy.ndarray): The input data array. @@ -428,7 +432,8 @@ class ExponentialFitter: Methods: compute_expected_distribution(norm, loc, scale): - Computes the expected distribution given the normalization, location, and scale parameters. + Computes the expected distribution given the normalization, location, and\ + scale parameters. expected_distribution(x): Returns the expected distribution given the parameters in `x`. compute_minus2loglike(x): @@ -466,8 +471,7 @@ def compute_minus2loglike(self, x): def pois(x, A, R): - """ - Computes the expected distribution for a Poisson process with rate parameter `R`. + """Computes the expected distribution for a Poisson process with rate parameter `R`. Args: x (float): The input value. @@ -477,13 +481,12 @@ def pois(x, A, R): Returns: float: The expected distribution for the Poisson process. """ - """poisson function, parameter r (rate) is the fit parameter""" + """Poisson function, parameter r (rate) is the fit parameter.""" return A * np.exp(x * R) def deadtime_and_expo_fit(time_tot, deadtime_us, run, output_plot=None): - """ - Computes the deadtime and exponential fit parameters for a given dataset. + """Computes the deadtime and exponential fit parameters for a given dataset. Args: time_tot (float): The total time of the dataset. @@ -579,8 +582,9 @@ def deadtime_and_expo_fit(time_tot, deadtime_us, run, output_plot=None): ax.text( 600, entries[1] / 1, - "$y = A \cdot \exp({-R \cdot x})$\n" - # + r'$A=%2.2f \pm %2.2f$'%(as_si((result.params['A'].value/1000)*1e3,2), as_si((result.params['A'].stderr/1000)*1e3,2)) + r"$y = A \cdot \exp({-R \cdot x})$\n" + # + r'$A=%2.2f \pm %2.2f$'%(as_si((result.params['A'].value/1000) + # *1e3,2), as_si((result.params['A'].stderr/1000)*1e3,2)) + r"$A=%2.2f \pm %2.2f$" % (result.params["A"].value, result.params["A"].stderr) + "\n" From 2127d0ff8e6eb989a84e7e11ab3839bceee98488 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 20 Jan 2025 20:31:23 +0100 Subject: [PATCH 17/71] fix imports --- src/nectarchain/trr_test_suite/__init__.py | 5 +++++ src/nectarchain/trr_test_suite/gui.py | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/trr_test_suite/__init__.py b/src/nectarchain/trr_test_suite/__init__.py index e69de29b..417f98f8 100644 --- a/src/nectarchain/trr_test_suite/__init__.py +++ b/src/nectarchain/trr_test_suite/__init__.py @@ -0,0 +1,5 @@ +from .gui import TestRunner + +__all__ = [ + "TestRunner", +] diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index cab2fa32..92ce0f63 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -19,17 +19,11 @@ import sys import tempfile -import deadtime -import linearity -import pedestal -import pix_couple_tim_uncertainty -import pix_tim_uncertainty -import trigger_timing from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt6.QtCore import QProcess, QTimer -from PyQt6.QtWidgets import ( +from PyQt5.QtCore import QProcess, QTimer +from PyQt5.QtWidgets import ( QApplication, QComboBox, QGroupBox, @@ -46,6 +40,15 @@ QWidgetItem, ) +import nectarchain.trr_test_suite.deadtime as deadtime +import nectarchain.trr_test_suite.linearity as linearity +import nectarchain.trr_test_suite.pedestal as pedestal +import nectarchain.trr_test_suite.pix_tim_uncertainty as pix_tim_uncertainty +import nectarchain.trr_test_suite.trigger_timing as trigger_timing +from nectarchain.trr_test_suite import ( + pix_couple_tim_uncertainty as pix_couple_tim_uncertainty, +) + # Ensure the src directory is in sys.path test_dir = os.path.abspath("src") if test_dir not in sys.path: From 3c8e2030f18ea384d8ef90c5760f9d787cd48084 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 22 Jan 2025 15:44:01 +0100 Subject: [PATCH 18/71] fix imports and format gange pyq5 to pyqt6 --- src/nectarchain/trr_test_suite/deadtime.py | 8 ++++++-- src/nectarchain/trr_test_suite/gui.py | 16 ++++++++++------ src/nectarchain/trr_test_suite/linearity.py | 4 ++-- src/nectarchain/trr_test_suite/pedestal.py | 4 ++-- .../trr_test_suite/pix_couple_tim_uncertainty.py | 9 +++++---- .../trr_test_suite/pix_tim_uncertainty.py | 5 +++-- .../trr_test_suite/tools_components.py | 2 +- src/nectarchain/trr_test_suite/trigger_timing.py | 5 +++-- 8 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/nectarchain/trr_test_suite/deadtime.py b/src/nectarchain/trr_test_suite/deadtime.py index 5c05add3..c01a5c27 100644 --- a/src/nectarchain/trr_test_suite/deadtime.py +++ b/src/nectarchain/trr_test_suite/deadtime.py @@ -10,8 +10,12 @@ from astropy import units as u from iminuit import Minuit -from .tools_components import DeadtimeTestTool -from .utils import ExponentialFitter, deadtime_labels, source_ids_deadtime +from nectarchain.trr_test_suite.tools_components import DeadtimeTestTool +from nectarchain.trr_test_suite.utils import ( + ExponentialFitter, + deadtime_labels, + source_ids_deadtime, +) def get_args(): diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 92ce0f63..d49c89aa 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -19,11 +19,11 @@ import sys import tempfile -from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar +from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt5.QtCore import QProcess, QTimer -from PyQt5.QtWidgets import ( +from PyQt6.QtCore import QProcess, QTimer +from PyQt6.QtWidgets import ( QApplication, QComboBox, QGroupBox, @@ -182,7 +182,9 @@ def init_ui(self): # Add a stretchable spacer to push the canvas to the right top_layout.addSpacerItem( - QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum) + QSpacerItem( + 40, 20, QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Minimum + ) ) # Create a vertical layout for the plot container @@ -405,7 +407,9 @@ def run_test(self): self.output_text_edit.clear() self.process = QProcess(self) - self.process.setProcessChannelMode(QProcess.MergedChannels) + self.process.setProcessChannelMode( + QProcess.ProcessChannelMode.MergedChannels + ) self.process.readyReadStandardOutput.connect(self.read_process_output) self.process.finished.connect(self.process_finished) @@ -540,4 +544,4 @@ def cleanup_tempdir(self): if __name__ == "__main__": app = QApplication(sys.argv) ex = TestRunner() - sys.exit(app.exec_()) + sys.exit(app.exec()) diff --git a/src/nectarchain/trr_test_suite/linearity.py b/src/nectarchain/trr_test_suite/linearity.py index b62461d0..1d05c8d2 100644 --- a/src/nectarchain/trr_test_suite/linearity.py +++ b/src/nectarchain/trr_test_suite/linearity.py @@ -9,8 +9,8 @@ import numpy as np from lmfit.models import Model -from .tools_components import LinearityTestTool -from .utils import ( +from nectarchain.trr_test_suite.tools_components import LinearityTestTool +from nectarchain.trr_test_suite.utils import ( err_ratio, err_sum, linear_fit_function, diff --git a/src/nectarchain/trr_test_suite/pedestal.py b/src/nectarchain/trr_test_suite/pedestal.py index 17185a38..58abc9eb 100644 --- a/src/nectarchain/trr_test_suite/pedestal.py +++ b/src/nectarchain/trr_test_suite/pedestal.py @@ -8,8 +8,8 @@ import matplotlib.pyplot as plt import numpy as np -from .tools_components import PedestalTool -from .utils import adc_to_pe, pe2photons +from nectarchain.trr_test_suite.tools_components import PedestalTool +from nectarchain.trr_test_suite.utils import adc_to_pe, pe2photons def get_args(): diff --git a/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py index ad599075..2138ae06 100644 --- a/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_couple_tim_uncertainty.py @@ -5,7 +5,8 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import ToMPairsTool + +from nectarchain.trr_test_suite.tools_components import ToMPairsTool def get_args(): @@ -54,9 +55,9 @@ def get_args(): type=str, help=".csv file with pmt transit time corrections", required=False, - default="../transit_time/\ - hv_pmt_tom_correction_laser_measurement_per_pixel_fit_\ - sqrt_hv_newmethod.csv", + default="../transit_time/" + "hv_pmt_tom_correction_laser_measurement_per_pixel_fit" + "sqrt_hv_newmethod.csv", ) parser.add_argument( "-o", diff --git a/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py index d4a6c181..53c644c1 100644 --- a/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py +++ b/src/nectarchain/trr_test_suite/pix_tim_uncertainty.py @@ -7,8 +7,9 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import TimingResolutionTestTool -from utils import pe2photons, photons2pe + +from nectarchain.trr_test_suite.tools_components import TimingResolutionTestTool +from nectarchain.trr_test_suite.utils import pe2photons, photons2pe def get_args(): diff --git a/src/nectarchain/trr_test_suite/tools_components.py b/src/nectarchain/trr_test_suite/tools_components.py index 158ce4c6..5154c93d 100644 --- a/src/nectarchain/trr_test_suite/tools_components.py +++ b/src/nectarchain/trr_test_suite/tools_components.py @@ -12,11 +12,11 @@ from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from scipy.interpolate import InterpolatedUnivariateSpline from scipy.signal import find_peaks -from utils import adc_to_pe, argmedian from nectarchain.data.container import NectarCAMContainer from nectarchain.makers import EventsLoopNectarCAMCalibrationTool from nectarchain.makers.component import NectarCAMComponent +from nectarchain.trr_test_suite.utils import adc_to_pe, argmedian # overriding so we can have maxevents in the path diff --git a/src/nectarchain/trr_test_suite/trigger_timing.py b/src/nectarchain/trr_test_suite/trigger_timing.py index 0d679959..933cf03e 100644 --- a/src/nectarchain/trr_test_suite/trigger_timing.py +++ b/src/nectarchain/trr_test_suite/trigger_timing.py @@ -7,8 +7,9 @@ import matplotlib.pyplot as plt import numpy as np -from tools_components import TriggerTimingTestTool -from utils import pe2photons + +from nectarchain.trr_test_suite.tools_components import TriggerTimingTestTool +from nectarchain.trr_test_suite.utils import pe2photons def get_args(): From 5c06ef0f1cf7afdef9663143346f22910785a0d6 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 22 Jan 2025 15:50:48 +0100 Subject: [PATCH 19/71] add h5py depencie --- environment.yml | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 7d79cf2f..6240c9e1 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,9 @@ dependencies: - sphinx - sphinx-automodapi - pydata-sphinx-theme - - lmfit + - lmfit # needed into TRR + - h5py # needed into TRR (should be removed to use I/O methods of containers) + - pyqt # [linux] - pip: - zeo - zodb diff --git a/pyproject.toml b/pyproject.toml index d5eb3148..bafbc220 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "scipy==1.11.4", "zodb", "zeo", + "h5py", "pyqt6", "pyqtgraph", ] From 01fab180e577ebccfa97a0d6132a8b30e86f9e68 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 22 Jan 2025 15:54:51 +0100 Subject: [PATCH 20/71] add lmfit to pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index bafbc220..79deaba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "scipy==1.11.4", "zodb", "zeo", + "lmfit", "h5py", "pyqt6", "pyqtgraph", From 9317102abfc060a274bada08f4739563b692ac46 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 00:16:03 +0100 Subject: [PATCH 21/71] fix docstring --- src/nectarchain/trr_test_suite/gui.py | 8 +++----- src/nectarchain/trr_test_suite/linearity.py | 10 +++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index d49c89aa..7e42298e 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -1,6 +1,5 @@ -"""The `TestRunner` class is a GUI application that allows the user to run various tests -and display the results. - +"""The ``TestRunner`` class is a GUI application that allows the\ + user to run various tests and display the results. The class provides the following functionality: - Allows the user to select a test from a dropdown menu. - Dynamically generates input fields based on the selected test. @@ -8,7 +7,6 @@ - Displays the test results in a plot canvas, with navigation buttons to switch between\ multiple plots. - Provides a dark-themed UI with custom styling for various UI elements. - The class uses the PyQt5 library for the GUI implementation and the Matplotlib library\ for plotting the test results. """ @@ -129,7 +127,7 @@ def init_ui(self): QPushButton:hover { background-color: #45a049; /* Darker green on hover */ } - """ + """ ) # Horizontal layout for test options (left) and plot canvas (right) diff --git a/src/nectarchain/trr_test_suite/linearity.py b/src/nectarchain/trr_test_suite/linearity.py index 1d05c8d2..ab4a6cd5 100644 --- a/src/nectarchain/trr_test_suite/linearity.py +++ b/src/nectarchain/trr_test_suite/linearity.py @@ -83,11 +83,11 @@ def get_args(): def main(): - """The `main()` function is the entry point of the linearity test script. It parses - the command-line arguments, processes the specified runs, and generates plots to - visualize the linearity and charge resolution of the detector. The function performs - the following key steps: - + """ + The `main()` function is the entry point of the linearity test script. It parses \ + the command-line arguments, processes the specified runs, and generates plots\ + to visualize the linearity and charge resolution of the detector. The\ + function performs the following key steps: 1. Parses the command-line arguments using the `get_args()` function, which sets up\ the argument parser and handles the input parameters. 2. Iterates through the specified run list, processing each run using the\ From 928def7d617ddb102bc25c69b65315498e3a3a2b Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 17:06:10 +0100 Subject: [PATCH 22/71] fix sphinx issue with PyQt6 subclass imports --- src/nectarchain/trr_test_suite/gui.py | 67 +++++++++++++++------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 7e42298e..6cd16170 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -1,16 +1,3 @@ -"""The ``TestRunner`` class is a GUI application that allows the\ - user to run various tests and display the results. -The class provides the following functionality: -- Allows the user to select a test from a dropdown menu. -- Dynamically generates input fields based on the selected test. -- Runs the selected test and displays the output in a text box. -- Displays the test results in a plot canvas, with navigation buttons to switch between\ - multiple plots. -- Provides a dark-themed UI with custom styling for various UI elements. -The class uses the PyQt5 library for the GUI implementation and the Matplotlib library\ - for plotting the test results. -""" - import argparse import os import pickle @@ -20,23 +7,30 @@ from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt6.QtCore import QProcess, QTimer -from PyQt6.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, -) + +if "sphinx" not in sys.modules: + from PyQt6.QtCore import QProcess, QTimer + from PyQt6.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, + ) +else: + + class QWidget: + pass + import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity @@ -56,6 +50,19 @@ class TestRunner(QWidget): + """The ``TestRunner`` class is a GUI application that allows the\ + user to run various tests and display the results. + The class provides the following functionality: + - Allows the user to select a test from a dropdown menu. + - Dynamically generates input fields based on the selected test. + - Runs the selected test and displays the output in a text box. + - Displays the test results in a plot canvas, with navigation buttons\ + to switch between multiple plots. + - Provides a dark-themed UI with custom styling for various UI elements. + The class uses the PyQt5 library for the GUI implementation and the Matplotlib\ + library for plotting the test results. + """ + test_modules = { "Linearity Test": linearity, "Deadtime Test": deadtime, From f34add1787868d69f6f21838846f9da45b8ec849 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:02:39 +0100 Subject: [PATCH 23/71] Revert "Merge branch 'main' into TTR_tests" This reverts commit d07747ab1b0b976358ad4cd9621891c1f24b699b, reversing changes made to 2dadf7e04b6b6d2ec37f008cf42792c41fa7433c. --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79192908..3adeca1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,10 +144,7 @@ jobs: run: | pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts - - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - uses: codecov/codecov-action@v5 docs: From a4a1764827374b5ca32ad876a7ee703e511bd3aa Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:39:08 +0100 Subject: [PATCH 24/71] realign to main --- environment.yml | 2 +- .../makers/component/tests/test_flatfield_spe_component.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 6240c9e1..6706e6b0 100644 --- a/environment.yml +++ b/environment.yml @@ -25,5 +25,5 @@ dependencies: - zodb - mechanize - browser-cookie3 + - pyqt6 - pyqtgraph - - pyqt6 # [osx and arm64] diff --git a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py index bb5eae67..fb0a8aff 100644 --- a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py +++ b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py @@ -71,7 +71,6 @@ def test_call(self, instance, event): instance(event) assert len(instance.chargesComponent.trigger_list) == 1 - # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): SPEalgorithm.window_length.default_value = 2 # We need to re-instance objects because otherwise, en exception is raised : @@ -188,7 +187,6 @@ def test_init(self, instance): assert isinstance(instance.chargesComponent, ChargesComponent) assert instance._chargesContainers is None - # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): # We need to re-instance objects because otherwise, en exception is raised : # ReferenceError('weakly-referenced object no longer exists') From 784432c81741a0f51f59eed7d3d00e53f9ae5961 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:40:41 +0100 Subject: [PATCH 25/71] fix to main --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6706e6b0..6240c9e1 100644 --- a/environment.yml +++ b/environment.yml @@ -25,5 +25,5 @@ dependencies: - zodb - mechanize - browser-cookie3 - - pyqt6 - pyqtgraph + - pyqt6 # [osx and arm64] From 325c0cef31379f207c441906a0afb7e37cc3e2ad Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 23 Jan 2025 21:52:02 +0100 Subject: [PATCH 26/71] reparing unwanted changes --- .github/workflows/ci.yml | 5 ++++- .../makers/component/tests/test_flatfield_spe_component.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3adeca1a..79192908 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,7 +144,10 @@ jobs: run: | pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts - - uses: codecov/codecov-action@v5 + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v5 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} docs: diff --git a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py index fb0a8aff..bb5eae67 100644 --- a/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py +++ b/src/nectarchain/makers/component/tests/test_flatfield_spe_component.py @@ -71,6 +71,7 @@ def test_call(self, instance, event): instance(event) assert len(instance.chargesComponent.trigger_list) == 1 + # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): SPEalgorithm.window_length.default_value = 2 # We need to re-instance objects because otherwise, en exception is raised : @@ -187,6 +188,7 @@ def test_init(self, instance): assert isinstance(instance.chargesComponent, ChargesComponent) assert instance._chargesContainers is None + # @pytest.mark.skip(reason="test multiproc make GitHub worker be killed") def test_finish_multiproc(self): # We need to re-instance objects because otherwise, en exception is raised : # ReferenceError('weakly-referenced object no longer exists') From d3e11ad04faf246f15ebed4083baac7070732818 Mon Sep 17 00:00:00 2001 From: jlenain Date: Thu, 30 Jan 2025 22:25:45 +0100 Subject: [PATCH 27/71] Fix Sphinx intermapping with PyQt documentation --- docs/conf.py | 1 + environment.yml | 1 + pyproject.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 024296e0..6e4f71a5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,6 +51,7 @@ # class signature) "numpydoc", "sphinx_design", + "sphinx_qt_documentation", ] autosummary_generate = True # Turn on sphinx.ext.autosummary diff --git a/environment.yml b/environment.yml index 6240c9e1..feb99e09 100644 --- a/environment.yml +++ b/environment.yml @@ -27,3 +27,4 @@ dependencies: - browser-cookie3 - pyqtgraph - pyqt6 # [osx and arm64] + - sphinx-qt-documentation diff --git a/pyproject.toml b/pyproject.toml index 79deaba9..5a930987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ docs = [ "sphinx-autodoc-typehints", "sphinx-automodapi", "sphinx-design", + "sphinx-qt-documentation", "pydata_sphinx_theme", "numpydoc", "tomli; python_version < '3.11'" From dd5d6210d1cea09baa5afa8c0f77929bf3bd98dd Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 31 Jan 2025 10:51:07 +0100 Subject: [PATCH 28/71] remove import fix for sphinx --- src/nectarchain/trr_test_suite/gui.py | 41 +++++++++++---------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 6cd16170..99feff31 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -7,30 +7,23 @@ from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure - -if "sphinx" not in sys.modules: - from PyQt6.QtCore import QProcess, QTimer - from PyQt6.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, - ) -else: - - class QWidget: - pass - +from PyQt6.QtCore import QProcess, QTimer +from PyQt6.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, +) import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity From 33399828e927a57ba85358a8dd913b0ec93891ea Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 31 Jan 2025 14:43:30 +0100 Subject: [PATCH 29/71] Switch back to PyQt5 instead of PyQt6, to be able to launch the TRR GUI, on Linux at least --- pyproject.toml | 3 ++- src/nectarchain/trr_test_suite/gui.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5a930987..4e3124e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ dependencies = [ "zeo", "lmfit", "h5py", - "pyqt6", + 'pyqt5 ; platform_system == "Linux"', + 'pyqt6 ; platform_system != "Linux"', # for macOS "pyqtgraph", ] diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 99feff31..1dea5516 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -4,11 +4,11 @@ import sys import tempfile -from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar +from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt6.QtCore import QProcess, QTimer -from PyQt6.QtWidgets import ( +from PyQt5.QtCore import QProcess, QTimer +from PyQt5.QtWidgets import ( QApplication, QComboBox, QGroupBox, From 3a4504f1af0930b26ccb96531556237bfa333a50 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 31 Jan 2025 16:36:36 +0100 Subject: [PATCH 30/71] To try to ensure that the TRR GUI is launchable both from Linux and macOS, add conditional imports on PyQt5/PyQt6. --- src/nectarchain/trr_test_suite/gui.py | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 1dea5516..36789451 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -4,26 +4,49 @@ import sys import tempfile -from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure -from PyQt5.QtCore import QProcess, QTimer -from PyQt5.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, -) + +try: + from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar + from PyQt6.QtCore import QProcess, QTimer + from PyQt6.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, + ) +except ImportError: + from PyQt5.QtCore import QProcess, QTimer + from PyQt5.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, + ) + from matplotlib.backends.backend_qt5 import ( + NavigationToolbar2QT as NavigationToolbar, + ) import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity From 50282e8a6fa097aa7c6a34dfb13001a6b09332ef Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 7 Feb 2025 17:54:58 +0100 Subject: [PATCH 31/71] fix pyqt5 dependencies of matplotlib when pyqt6 installation on ubuntu --- src/nectarchain/trr_test_suite/gui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 36789451..08ce81f6 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -4,11 +4,12 @@ import sys import tempfile -from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas +# from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure try: from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar + from matplotlib.backends.backend_qtagg import FigureCanvasQTAgg as FigureCanvas from PyQt6.QtCore import QProcess, QTimer from PyQt6.QtWidgets import ( QApplication, @@ -47,6 +48,7 @@ from matplotlib.backends.backend_qt5 import ( NavigationToolbar2QT as NavigationToolbar, ) + from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity From 8b3d74cab54a55e63981c80d4070085098c46b86 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 7 Feb 2025 18:31:26 +0100 Subject: [PATCH 32/71] very simple unit test for TRR --- .../trr_test_suite/tests/test_gui.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/nectarchain/trr_test_suite/tests/test_gui.py diff --git a/src/nectarchain/trr_test_suite/tests/test_gui.py b/src/nectarchain/trr_test_suite/tests/test_gui.py new file mode 100644 index 00000000..3c72edde --- /dev/null +++ b/src/nectarchain/trr_test_suite/tests/test_gui.py @@ -0,0 +1,23 @@ +from unittest.mock import MagicMock, patch + +from nectarchain.trr_test_suite import TestRunner + +try: + import PyQt6 # noqa: F401 + + pyqt_module = "PyQt6" +except ImportError: + import PyQt5 # noqa: F401 + + pyqt_module = "PyQt5" + + +class TestTestRunner: + @patch(f"{pyqt_module}.QtWidgets.QWidget.__init__", return_value=None) + @patch(f"{pyqt_module}.QtWidgets.QApplication", MagicMock()) + @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) + def test_init(self, mock_init_ui, mock_qwidget): + runner = TestRunner() + assert isinstance(runner, TestRunner) + mock_qwidget.assert_called_once() + mock_init_ui.assert_called_once() From 8fcfdde94ad19c3cfc2962d50bc185bfa9dc959e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 7 Feb 2025 20:52:27 +0100 Subject: [PATCH 33/71] fix strange behavior in tests --- src/nectarchain/makers/tests/test_core_makers.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/nectarchain/makers/tests/test_core_makers.py b/src/nectarchain/makers/tests/test_core_makers.py index d447ba8c..c84d2446 100644 --- a/src/nectarchain/makers/tests/test_core_makers.py +++ b/src/nectarchain/makers/tests/test_core_makers.py @@ -1,5 +1,4 @@ import logging -import os import pathlib from unittest.mock import MagicMock, patch @@ -58,22 +57,22 @@ class TestEventsLoopNectarCAMCalibrationTool(TestBaseNectarCAMCalibrationTool): @pytest.fixture def tool_instance(self): - return EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER) + return EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, + output_path=pathlib.Path("/tmp/test_output.h5"), + ) @pytest.fixture def tool_instance_run_file(self): return EventsLoopNectarCAMCalibrationTool( run_number=self.RUN_NUMBER, run_file=self.RUN_FILE, - output_path=pathlib.Path(f"/tmp/{np.random.random()}test_output.h5") + output_path=pathlib.Path(f"/tmp/{np.random.random()}test_output.h5"), # to avoid I/O conflicts between tests ) def test_init_output_path(self, tool_instance): - expected_path = pathlib.Path( - f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs" - f"/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5" - ) + expected_path = pathlib.Path("/tmp/test_output.h5") assert tool_instance.output_path == expected_path assert tool_instance.run_number == self.RUN_NUMBER assert tool_instance.max_events is None From 8f18c4562d4501b610bf78290454e05adb6060c1 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 7 Feb 2025 21:36:46 +0100 Subject: [PATCH 34/71] change test_gui to work on macos --- .../trr_test_suite/tests/test_gui.py | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/nectarchain/trr_test_suite/tests/test_gui.py b/src/nectarchain/trr_test_suite/tests/test_gui.py index 3c72edde..9d2f6de7 100644 --- a/src/nectarchain/trr_test_suite/tests/test_gui.py +++ b/src/nectarchain/trr_test_suite/tests/test_gui.py @@ -1,23 +1,36 @@ -from unittest.mock import MagicMock, patch +import unittest +from unittest.mock import patch from nectarchain.trr_test_suite import TestRunner -try: - import PyQt6 # noqa: F401 - pyqt_module = "PyQt6" -except ImportError: - import PyQt5 # noqa: F401 - - pyqt_module = "PyQt5" +class TestTestRunner(unittest.TestCase): + @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) + def test_init_pyqt(self, mock_init_ui): + try: + import PyQt6 # noqa: F401 + with patch( + "PyQt6.QtWidgets.QWidget.__init__", return_value=None + ) as mock_qwidget: + runner = TestRunner() + assert isinstance(runner, TestRunner) + mock_qwidget.assert_called_once() + mock_init_ui.assert_called_once() + except ImportError: + self.skipTest("PyQt6 is not available") -class TestTestRunner: - @patch(f"{pyqt_module}.QtWidgets.QWidget.__init__", return_value=None) - @patch(f"{pyqt_module}.QtWidgets.QApplication", MagicMock()) @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) - def test_init(self, mock_init_ui, mock_qwidget): - runner = TestRunner() - assert isinstance(runner, TestRunner) - mock_qwidget.assert_called_once() - mock_init_ui.assert_called_once() + def test_init_pyqt5(self, mock_init_ui): + try: + import PyQt5 # noqa: F401 + + with patch( + "PyQt5.QtWidgets.QWidget.__init__", return_value=None + ) as mock_qwidget: + runner = TestRunner() + assert isinstance(runner, TestRunner) + mock_qwidget.assert_called_once() + mock_init_ui.assert_called_once() + except ImportError: + self.skipTest("PyQt5 is not available") From 58c19ffe799ddcbfea77b102c0fd0e8761bd591d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 7 Feb 2025 21:56:59 +0100 Subject: [PATCH 35/71] mac os is shitty --- src/nectarchain/trr_test_suite/tests/test_gui.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/trr_test_suite/tests/test_gui.py b/src/nectarchain/trr_test_suite/tests/test_gui.py index 9d2f6de7..0e12ecae 100644 --- a/src/nectarchain/trr_test_suite/tests/test_gui.py +++ b/src/nectarchain/trr_test_suite/tests/test_gui.py @@ -8,11 +8,9 @@ class TestTestRunner(unittest.TestCase): @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) def test_init_pyqt(self, mock_init_ui): try: - import PyQt6 # noqa: F401 + from PyQt6.QtWidgets import QWidget # noqa: F401 - with patch( - "PyQt6.QtWidgets.QWidget.__init__", return_value=None - ) as mock_qwidget: + with patch("QWidget.__init__", return_value=None) as mock_qwidget: runner = TestRunner() assert isinstance(runner, TestRunner) mock_qwidget.assert_called_once() @@ -23,11 +21,9 @@ def test_init_pyqt(self, mock_init_ui): @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) def test_init_pyqt5(self, mock_init_ui): try: - import PyQt5 # noqa: F401 + from PyQt5.QtWidgets import QWidget # noqa: F401 - with patch( - "PyQt5.QtWidgets.QWidget.__init__", return_value=None - ) as mock_qwidget: + with patch("QWidget.__init__", return_value=None) as mock_qwidget: runner = TestRunner() assert isinstance(runner, TestRunner) mock_qwidget.assert_called_once() From 5c7e397c19cd2669591a3700726ec36c00b02a9c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 11:46:46 +0100 Subject: [PATCH 36/71] remove pyqt6 --- .github/workflows/ci.yml | 6 ------ environment.yml | 3 +-- pyproject.toml | 3 +-- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79192908..8562c744 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,12 +115,6 @@ jobs: sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev # start xvfb in the background sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & - - - name: Instal QT and setup environment on macOS - if: matrix.os == 'macos-14' - run: | - brew install pyqt@6 - echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV - name: Install dependencies env: diff --git a/environment.yml b/environment.yml index feb99e09..bfdf154c 100644 --- a/environment.yml +++ b/environment.yml @@ -19,12 +19,11 @@ dependencies: - pydata-sphinx-theme - lmfit # needed into TRR - h5py # needed into TRR (should be removed to use I/O methods of containers) - - pyqt # [linux] + - pyqt - pip: - zeo - zodb - mechanize - browser-cookie3 - pyqtgraph - - pyqt6 # [osx and arm64] - sphinx-qt-documentation diff --git a/pyproject.toml b/pyproject.toml index 4e3124e4..5673db15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,7 @@ dependencies = [ "zeo", "lmfit", "h5py", - 'pyqt5 ; platform_system == "Linux"', - 'pyqt6 ; platform_system != "Linux"', # for macOS + "pyqt5", "pyqtgraph", ] From 78e3058193aa581ba3bc90f8d2df3871d5384ab5 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 11:50:58 +0100 Subject: [PATCH 37/71] install pyqt5 in CI for mac --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8562c744..d42588c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,12 @@ jobs: sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev # start xvfb in the background sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & + + - name: Instal QT and setup environment on macOS + #if: matrix.os == 'macos-14' + run: | + brew install pyqt@5 + echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV - name: Install dependencies env: From af90fd30864464109650721f0ea76f4588234281 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 11:52:29 +0100 Subject: [PATCH 38/71] typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d42588c4..f01c23e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & - name: Instal QT and setup environment on macOS - #if: matrix.os == 'macos-14' + if: matrix.os == 'macos-14' run: | brew install pyqt@5 echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV From c5010e27b1aa4ad359be16673d24576aa6dd3b77 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 12:01:34 +0100 Subject: [PATCH 39/71] sqlite issue --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f01c23e0..37972d3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,8 @@ jobs: - name: Instal QT and setup environment on macOS if: matrix.os == 'macos-14' run: | + brew install sqlite + brew link --force --overwrite sqlite brew install pyqt@5 echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV From 6e1128b7f87c6810e4a8cdfd78880011cbe8c638 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 12:55:41 +0100 Subject: [PATCH 40/71] macos depencies bzip2 --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37972d3d..e0076733 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,8 +119,7 @@ jobs: - name: Instal QT and setup environment on macOS if: matrix.os == 'macos-14' run: | - brew install sqlite - brew link --force --overwrite sqlite + brew install bzip2 readline ncurses brew install pyqt@5 echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV From f2ac6da9a414f8f1c33ae0ca84d1d4f487efe40f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:06:08 +0100 Subject: [PATCH 41/71] check pyqt installation --- .github/workflows/ci.yml | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0076733..81035a6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,23 @@ jobs: run: | sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml + - name: Setup xvfb for X11 display + if: matrix.os == 'ubuntu-latest' + run: | + # Stuff copied wildly from several stackoverflow posts + sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 libxcb-shape0 libglib2.0-0 libgl1-mesa-dev + sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev + # start xvfb in the background + sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & + + - name: Instal QT and setup environment on macOS + if: matrix.os == 'macos-14' + run: | + #brew install bzip2 readline ncurses + brew install qt + brew install pyqt@5 + echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV + - name: Mamba setup if: matrix.install-method == 'mamba' uses: mamba-org/setup-micromamba@v2 @@ -107,21 +124,6 @@ jobs: python-version: ${{ matrix.python-version }} check-latest: true - - name: Setup xvfb for X11 display - if: matrix.os == 'ubuntu-latest' - run: | - # Stuff copied wildly from several stackoverflow posts - sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 libxcb-shape0 libglib2.0-0 libgl1-mesa-dev - sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev - # start xvfb in the background - sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & - - - name: Instal QT and setup environment on macOS - if: matrix.os == 'macos-14' - run: | - brew install bzip2 readline ncurses - brew install pyqt@5 - echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV - name: Install dependencies env: @@ -137,9 +139,10 @@ jobs: echo "pip install -e ." pip install -e .[test] - - name: Check pyqt version + - name: Check pyqt version and intallation run: | pip list | grep PyQt + ldd $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)") - name: Tests run: | From 454dcdcf9f04eae46bb215299955ee94cc018180 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:15:46 +0100 Subject: [PATCH 42/71] test --- .github/workflows/ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81035a6b..69eccc22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,9 +105,9 @@ jobs: if: matrix.os == 'macos-14' run: | #brew install bzip2 readline ncurses - brew install qt - brew install pyqt@5 - echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV + #brew install qt + #brew install pyqt@5 + #echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV - name: Mamba setup if: matrix.install-method == 'mamba' @@ -142,7 +142,13 @@ jobs: - name: Check pyqt version and intallation run: | pip list | grep PyQt - ldd $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)") + + - name: Check mac-os installation + if: matrix.os == 'macos-14' + run: | + micromamba list -n ci | grep qt + micromamba list -n ci | grep pyqt + otool -L $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)") - name: Tests run: | From 3cfcb589d2e14b542369e57b32e1757ca1470bfa Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:20:43 +0100 Subject: [PATCH 43/71] remove macos dedicated installation --- .github/workflows/ci.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69eccc22..8bb3386b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,14 +100,7 @@ jobs: sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev # start xvfb in the background sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & - - - name: Instal QT and setup environment on macOS - if: matrix.os == 'macos-14' - run: | - #brew install bzip2 readline ncurses - #brew install qt - #brew install pyqt@5 - #echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV + - name: Mamba setup if: matrix.install-method == 'mamba' From bd9418bf2ca06ee291f5e9759435997b52d702ed Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:24:46 +0100 Subject: [PATCH 44/71] macos missing activation of environement in CI checker --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bb3386b..022aaabb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: if: matrix.os == 'macos-14' run: | micromamba list -n ci | grep qt - micromamba list -n ci | grep pyqt + mamba activate ci otool -L $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)") - name: Tests From c571b371872b9bbefaf5443fdbd3bda7de956ba9 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:36:43 +0100 Subject: [PATCH 45/71] try --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 022aaabb..97d1d85c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,6 +140,7 @@ jobs: if: matrix.os == 'macos-14' run: | micromamba list -n ci | grep qt + eval "$(micromamba shell hook --shell=bash)" mamba activate ci otool -L $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)") From 8a14fbf67c1167dd459a2d8a7dedbee5ed235441 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:39:58 +0100 Subject: [PATCH 46/71] add ssh debugger of CI (test) --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97d1d85c..d3db09ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,9 @@ jobs: with: fetch-depth: 0 + - name: Start SSH Debugging Session + uses: mxschmitt/action-tmate@v3 + - name: Set Python version env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -141,7 +144,7 @@ jobs: run: | micromamba list -n ci | grep qt eval "$(micromamba shell hook --shell=bash)" - mamba activate ci + micromamba activate ci otool -L $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)") - name: Tests From f8c94f77454ec6ea36eb296948e808c475736636 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:47:48 +0100 Subject: [PATCH 47/71] deteched ssh debug --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3db09ce..ec7c7385 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,8 @@ jobs: - name: Start SSH Debugging Session uses: mxschmitt/action-tmate@v3 + with: + detached: true - name: Set Python version env: From ea44cb664e54df93a2048b2c75f1331807e4e615 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:50:45 +0100 Subject: [PATCH 48/71] set up ssh debugger only on failure --- .github/workflows/ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec7c7385..84cf238c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,11 +86,6 @@ jobs: with: fetch-depth: 0 - - name: Start SSH Debugging Session - uses: mxschmitt/action-tmate@v3 - with: - detached: true - - name: Set Python version env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -158,6 +153,12 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - name: Start SSH Debugging Session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + detached: true + docs: needs: lint From 91ea35f0f898ed08a3e06ae739d6e74f95233082 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 13:58:07 +0100 Subject: [PATCH 49/71] remove detached --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84cf238c..fb1c4e0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,8 +156,6 @@ jobs: - name: Start SSH Debugging Session if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 - with: - detached: true docs: From 59458979eff606e42bcd4adf5ec7b7dc9fb0c21f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 14:12:39 +0100 Subject: [PATCH 50/71] install pyqt with brew before mamba installation on mac --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb1c4e0b..9bef5f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,10 @@ jobs: sudo apt-get update sudo apt-get install freeglut3-dev + - name: Install dependencies for Qt on mac-OS + if: matrix.os == 'macos-14' + run: | + brew install pyqt - uses: actions/checkout@v4 with: fetch-depth: 0 From 7d58e5955ce73cd2d65db03a4862d74ab5ced1d5 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 15:30:05 +0100 Subject: [PATCH 51/71] add --no-deps to pip insall nectarchain for pytest --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bef5f02..902f7fcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" echo "pip install -e ." - pip install -e .[test] + pip install -e --no-deps .[test] - name: Check pyqt version and intallation run: | From f32e2d232d422980b99824641099e36b619139f6 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 16:06:38 +0100 Subject: [PATCH 52/71] remove -e --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 902f7fcf..4ec0cb49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,8 +133,8 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - echo "pip install -e ." - pip install -e --no-deps .[test] + echo "pip install --no-deps .[test]" + pip install --no-deps .[test] - name: Check pyqt version and intallation run: | From b99b76297985875baf2194715115d6e43b533d92 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 16:11:17 +0100 Subject: [PATCH 53/71] --no-deps incompatible with -e --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ec0cb49..16748a5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,8 +133,8 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - echo "pip install --no-deps .[test]" - pip install --no-deps .[test] + echo "pip install -e --ignore-installed .[test]" + pip install -e --ignore-installed .[test] - name: Check pyqt version and intallation run: | From 4b51e1bc770c9f46dc17abb5d8720773c3b116b4 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 16:18:24 +0100 Subject: [PATCH 54/71] the order was wrong.. add --no-deps --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16748a5c..e2559ea0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,8 +133,8 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - echo "pip install -e --ignore-installed .[test]" - pip install -e --ignore-installed .[test] + echo "pip install --no-deps -e .[test]" + pip install --no-deps -e .[test] - name: Check pyqt version and intallation run: | From 9e8176b6cc1f5b619554afa31e3ceaba6934c16d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 16:38:16 +0100 Subject: [PATCH 55/71] terminator solution --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2559ea0..04485f79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,8 +133,8 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - echo "pip install --no-deps -e .[test]" - pip install --no-deps -e .[test] + echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" + pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))") - name: Check pyqt version and intallation run: | From e640cf0b9a7f9836cd7e3d9aacc52db1b62b49ad Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 16:51:49 +0100 Subject: [PATCH 56/71] format --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04485f79..7fe604b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,8 +133,8 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" - pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))") + #echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" + pip install -r <(python -c "import tomllib; print(str('\n').join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))") - name: Check pyqt version and intallation run: | From 872d9f89d8ca37ce330c79770d20033b68dae8f3 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 17:15:38 +0100 Subject: [PATCH 57/71] macos shell --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fe604b5..fcee468c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" #echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" - pip install -r <(python -c "import tomllib; print(str('\n').join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))") + eval "$(pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))"))" - name: Check pyqt version and intallation run: | From 0e7d2b750ca82104f35e88f27c8e06d0e9754941 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 17:28:47 +0100 Subject: [PATCH 58/71] gozilla vs terminator --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcee468c..8878edb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,8 @@ jobs: "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" #echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" - eval "$(pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))"))" + python -c "import tomllib, sys; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install + - name: Check pyqt version and intallation run: | From c6e8851581799a1078a75a70c1e309d4ef3fe451 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 17:36:40 +0100 Subject: [PATCH 59/71] add cleaner pip installation of nectarchain --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8878edb0..d7da40cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,11 @@ jobs: #echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" python -c "import tomllib, sys; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install - + - name: Pip setup + if: matrix.install-method == 'pip' + run: | + pip install . + - name: Check pyqt version and intallation run: | pip list | grep PyQt From 380f9ca8aee6b0139f38d7ece9a7babd2e7646fe Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 17:42:50 +0100 Subject: [PATCH 60/71] clean separation of the instalation through pip and mamba --- .github/workflows/ci.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7da40cf..9caf98c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,25 +125,29 @@ jobs: - name: Install dependencies env: CTAPIPE_VERSION: ${{ matrix.ctapipe-version }} - run: | python --version echo "Installing additional pip packages" # we install ctapipe using pip to be able to select any commit, e.g. the current main pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - - #echo "pip install -r <(python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))")" - python -c "import tomllib, sys; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install - - - name: Pip setup - if: matrix.install-method == 'pip' - run: | - pip install . + + #nectarchain installation with pip + if [ "${{ matrix.install-method }}" == "pip" ]; then + pip install -e . + else + pip install -I -e . + fi - name: Check pyqt version and intallation run: | - pip list | grep PyQt + if [ "${{ matrix.install-method }}" == "pip" ]; then + pip list | grep PyQt + else + micromamba list -n ci qt + pip list | grep PyQt + fi + - name: Check mac-os installation if: matrix.os == 'macos-14' @@ -155,6 +159,8 @@ jobs: - name: Tests run: | + # install with pip dependencies for pytest + python -c "import tomllib, sys; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts - name: Upload coverage reports to Codecov with GitHub Action From 882df3355aacd79f5a5825f901383bc467a4022c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 17:49:47 +0100 Subject: [PATCH 61/71] tomlib -> toml for python < 3.11 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9caf98c7..76fe2f90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,13 +131,15 @@ jobs: # we install ctapipe using pip to be able to select any commit, e.g. the current main pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - + #nectarchain installation with pip if [ "${{ matrix.install-method }}" == "pip" ]; then pip install -e . else pip install -I -e . fi + # install with pip dependencies for pytest + python -c "import toml, sys; print('\n'.join(toml.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install - name: Check pyqt version and intallation run: | @@ -159,8 +161,6 @@ jobs: - name: Tests run: | - # install with pip dependencies for pytest - python -c "import tomllib, sys; print('\n'.join(tomllib.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts - name: Upload coverage reports to Codecov with GitHub Action From 8159e15de56f26c869143c79710b12ba68c9469f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 18:06:43 +0100 Subject: [PATCH 62/71] fix pip install --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76fe2f90..a45858c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,10 +136,11 @@ jobs: if [ "${{ matrix.install-method }}" == "pip" ]; then pip install -e . else - pip install -I -e . + pip install --no-deps -e . fi # install with pip dependencies for pytest - python -c "import toml, sys; print('\n'.join(toml.load(open('pyproject.toml', 'rb'))['project']['optional-dependencies']['test']))" | xargs pip install + pip install toml + python -c "import toml, sys; print('\n'.join(toml.load('pyproject.toml')['project']['optional-dependencies']['test']))" | xargs pip install - name: Check pyqt version and intallation run: | From 32cdbc2c4f509649b7b6fe3a4cddd202b712360f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 10 Feb 2025 18:40:51 +0100 Subject: [PATCH 63/71] clean --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a45858c6..d15b093e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,9 +138,9 @@ jobs: else pip install --no-deps -e . fi - # install with pip dependencies for pytest + # install with pip dependencies for pytest (NB : when we will use python > 3.10 for all configs, use tomllib instead of toml) pip install toml - python -c "import toml, sys; print('\n'.join(toml.load('pyproject.toml')['project']['optional-dependencies']['test']))" | xargs pip install + python -c "import toml, sys; print('\n'.join(toml.load('pyproject.toml')['project']['optional-dependencies']['test']))" | xargs pip install - name: Check pyqt version and intallation run: | From 06da9b022ba7d3e2d0f7f29388c0e0ccecbc140e Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 10:52:03 +0100 Subject: [PATCH 64/71] Remove any use of PyQt6 in TRR GUI --- src/nectarchain/trr_test_suite/gui.py | 63 ++++++------------- .../trr_test_suite/tests/test_gui.py | 13 ---- 2 files changed, 20 insertions(+), 56 deletions(-) diff --git a/src/nectarchain/trr_test_suite/gui.py b/src/nectarchain/trr_test_suite/gui.py index 08ce81f6..1ea62197 100644 --- a/src/nectarchain/trr_test_suite/gui.py +++ b/src/nectarchain/trr_test_suite/gui.py @@ -4,51 +4,28 @@ import sys import tempfile +from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar +from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas + # from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure - -try: - from matplotlib.backends.backend_qt import NavigationToolbar2QT as NavigationToolbar - from matplotlib.backends.backend_qtagg import FigureCanvasQTAgg as FigureCanvas - from PyQt6.QtCore import QProcess, QTimer - from PyQt6.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, - ) -except ImportError: - from PyQt5.QtCore import QProcess, QTimer - from PyQt5.QtWidgets import ( - QApplication, - QComboBox, - QGroupBox, - QHBoxLayout, - QLabel, - QLineEdit, - QMessageBox, - QPushButton, - QSizePolicy, - QSpacerItem, - QTextEdit, - QVBoxLayout, - QWidget, - QWidgetItem, - ) - from matplotlib.backends.backend_qt5 import ( - NavigationToolbar2QT as NavigationToolbar, - ) - from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas +from PyQt5.QtCore import QProcess, QTimer +from PyQt5.QtWidgets import ( + QApplication, + QComboBox, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QSizePolicy, + QSpacerItem, + QTextEdit, + QVBoxLayout, + QWidget, + QWidgetItem, +) import nectarchain.trr_test_suite.deadtime as deadtime import nectarchain.trr_test_suite.linearity as linearity diff --git a/src/nectarchain/trr_test_suite/tests/test_gui.py b/src/nectarchain/trr_test_suite/tests/test_gui.py index 0e12ecae..c332d45f 100644 --- a/src/nectarchain/trr_test_suite/tests/test_gui.py +++ b/src/nectarchain/trr_test_suite/tests/test_gui.py @@ -5,19 +5,6 @@ class TestTestRunner(unittest.TestCase): - @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) - def test_init_pyqt(self, mock_init_ui): - try: - from PyQt6.QtWidgets import QWidget # noqa: F401 - - with patch("QWidget.__init__", return_value=None) as mock_qwidget: - runner = TestRunner() - assert isinstance(runner, TestRunner) - mock_qwidget.assert_called_once() - mock_init_ui.assert_called_once() - except ImportError: - self.skipTest("PyQt6 is not available") - @patch("nectarchain.trr_test_suite.gui.TestRunner.init_ui", return_value=None) def test_init_pyqt5(self, mock_init_ui): try: From adaae13e43b9bebc7821c61a382d3f6cf4bff77c Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 11:10:54 +0100 Subject: [PATCH 65/71] Try to re-simplify the installation of dependencies for CI tests --- .github/workflows/ci.yml | 13 +++---------- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d15b093e..007fe427 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,16 +132,9 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - #nectarchain installation with pip - if [ "${{ matrix.install-method }}" == "pip" ]; then - pip install -e . - else - pip install --no-deps -e . - fi - # install with pip dependencies for pytest (NB : when we will use python > 3.10 for all configs, use tomllib instead of toml) - pip install toml - python -c "import toml, sys; print('\n'.join(toml.load('pyproject.toml')['project']['optional-dependencies']['test']))" | xargs pip install - + echo "pip install -e ." + pip install -e .[test] + - name: Check pyqt version and intallation run: | if [ "${{ matrix.install-method }}" == "pip" ]; then diff --git a/pyproject.toml b/pyproject.toml index 5673db15..1622b7b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dynamic = ["version"] [project.optional-dependencies] test = [ + "tomli; python_version < '3.11'" "pytest", "pytest-cov", "pytest-xdist", @@ -57,7 +58,6 @@ docs = [ "sphinx-qt-documentation", "pydata_sphinx_theme", "numpydoc", - "tomli; python_version < '3.11'" ] # we can use self-references to simplify all all = [ From b59dbce5ff1c2cf1c65cf68500450dd471dd25d6 Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 11:23:49 +0100 Subject: [PATCH 66/71] fix typo --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1622b7b0..bd562a90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dynamic = ["version"] [project.optional-dependencies] test = [ - "tomli; python_version < '3.11'" + "tomli; python_version < '3.11'", "pytest", "pytest-cov", "pytest-xdist", From 762d61c9b34c1d75d80af26d3ba843249483f683 Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 11:43:03 +0100 Subject: [PATCH 67/71] Re-introduce hacks for pip installation on macOS --- .github/workflows/ci.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 007fe427..d191b080 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,10 +132,22 @@ jobs: pip install \ "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - echo "pip install -e ." - pip install -e .[test] + if [ "${{ matrix.os }}" == "macos-14" ]; then + # nectarchain installation with pip + if [ "${{ matrix.install-method }}" == "pip" ]; then + pip install -e . + else + pip install --no-deps -e . + fi + # install with pip dependencies for pytest (NB : when we will use python > 3.10 for all configs, use tomllib instead of toml) + pip install toml + python -c "import toml, sys; print('\n'.join(toml.load('pyproject.toml')['project']['optional-dependencies']['test']))" | xargs pip install + else + echo "pip install -e ." + pip install -e .[test] + fi - - name: Check pyqt version and intallation + - name: Check pyqt version and installation run: | if [ "${{ matrix.install-method }}" == "pip" ]; then pip list | grep PyQt From 45368c9e71d25b3b511a5047e909260d47302fe4 Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 13:03:06 +0100 Subject: [PATCH 68/71] Try to fix tomli installation --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd562a90..a6fdfcdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "h5py", "pyqt5", "pyqtgraph", + "tomli; python_version < '3.11'", ] # needed for setuptools_scm, we don't define a static version @@ -42,7 +43,6 @@ dynamic = ["version"] [project.optional-dependencies] test = [ - "tomli; python_version < '3.11'", "pytest", "pytest-cov", "pytest-xdist", From 12739b5c4b9ff3ea13c9da723023e691c18db1e9 Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 13:13:31 +0100 Subject: [PATCH 69/71] cleanup CI workflow --- .github/workflows/ci.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d191b080..22b929d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,12 +4,10 @@ name: CI on: workflow_dispatch: push: - # TODO - # Remove comment: just a test to force CI on push - # branches: - # - main - # tags: - # - '**' + branches: + - main + tags: + - ""**" pull_request: env: @@ -70,6 +68,7 @@ jobs: env : DISPLAY: ":99.0" QT_SELECT: "qt5" + defaults: run: # We need login shells (-l) for micromamba to work. @@ -105,7 +104,6 @@ jobs: # start xvfb in the background sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & - - name: Mamba setup if: matrix.install-method == 'mamba' uses: mamba-org/setup-micromamba@v2 @@ -113,14 +111,12 @@ jobs: environment-name: "ci" environment-file: environment.yml - - name: Python setup if: matrix.install-method == 'pip' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} check-latest: true - - name: Install dependencies env: @@ -155,7 +151,6 @@ jobs: micromamba list -n ci qt pip list | grep PyQt fi - - name: Check mac-os installation if: matrix.os == 'macos-14' From da4027be33c3e76cdffa84a48c80c574b17ab20f Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 13:15:26 +0100 Subject: [PATCH 70/71] fix typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22b929d5..480d7dc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: branches: - main tags: - - ""**" + - "**" pull_request: env: From 60735e3e9ebd2524f64c2c15158c57569efdb2cb Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 11 Feb 2025 13:47:31 +0100 Subject: [PATCH 71/71] Disable SSH debugging in CI. Upload codecov report only for 1 platform/python version/installation method, not for all of them. --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 480d7dc5..45ae8cb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: python-version: "3.11" ctapipe-version: "v0.19.3" install-method: "pip" + extra-args: ["codecov"] # macos 14 image is arm64 based - os: "macos-14" python-version: "3.11" @@ -166,12 +167,13 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v5 + if: contains(matrix.extra-args, 'codecov') env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Start SSH Debugging Session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 + # - name: Start SSH Debugging Session + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 docs: