From 343977682541734e2890983027ce53b675fee4bb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Mon, 5 Feb 2024 17:29:33 +0100 Subject: [PATCH] Add a security to avoid submitting a job for a run for which the DQM was already run. (#107) Co-authored-by: Jean-Philippe Lenain --- .../dqm_job_submitter/submit_dqm_processor.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py index 1ec0ed2a..958ceb6e 100755 --- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py +++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py @@ -139,9 +139,29 @@ sys.exit(1) logger.info(f"Found SQLite files {sqlfilelist} in {dfcDir} and {dfcDirTomorrow}") +# Check already existing DQM outputs +dfcOutDir = "/vo.cta.in2p3.fr/user/j/jlenain/nectarcam/dqm" +infos = dfc.listDirectory(dfcOutDir) +if not infos["OK"] or not infos["Value"]["Successful"]: + logger.critical( + f"Could not properly retrieve the file metadata for {dfcOutDir} ... Exiting !" + ) + sys.exit(1) +metaOutDir = infos["Value"]["Successful"][dfcOutDir] + # Now, submit the DIRAC jobs: # for run in ['2721']: for run in runlist: + logger.debug(metaOutDir["Files"].keys()) + if f"{dfcOutDir}/NectarCAM_DQM_Run{run}.tar.gz" in metaOutDir["Files"].keys(): + logger.warning( + f"Run {run} already processed and present in {dfcOutDir}. If you really " + f"want to re-run the DQM on this run, consider deleting " + f"{dfcOutDir}/NectarCAM_DQM_Run{run}.tar.gz before executing this script. " + f"Skipping run {run} for now..." + ) + continue + j = Job() # j.setExecutable(f'{executable_wrapper}', '') j.setExecutable(f"{executable_wrapper}", f"-r {run}")