Eventdisplay · GernotMaier · Apr 9, 2024 · Apr 9, 2024 · Apr 9, 2024 · Apr 9, 2024
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,24 +1,36 @@
 repos:
   # https://pycqa.github.io/isort/docs/configuration/black_compatibility.html#integration-with-pre-commit
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.3.0
     hooks:
       - id: black
         args: ["--line-length=100"]
   # https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html?highlight=other%20tools#flake8
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
       - id: flake8
         args: ["--max-line-length=100", "--extend-ignore=E203,E712"]
   # https://github.com/pre-commit/pre-commit-hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+  # codespell
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        args: [
+          "--skip=*.json,*.bib,*.txt,*.cls,scripts/environment_vtscat_save.yml",
+          "--ignore-words-list=livetime",
+        ]
+        additional_dependencies:
+          - tomli
+
diff --git a/analysis/CTA.DISPTRAINING.qsub_analyse.sh b/analysis/CTA.DISPTRAINING.qsub_analyse.sh
@@ -1,12 +1,10 @@
 #!/bin/bash
 #
-# train disp for CTA
-#
-# called from CTA.DISPTRAINING_sub_analyse.sh
-#
+# train dispBDTs for CTA
 #
 
-ODIR=OFILE
+ODIR=OODIR
+DDIR=DDDIR
 RECID=RECONSTRUCTIONID
 TTYPE=TELTYPE
 BDT=MLPTYPE
@@ -17,15 +15,48 @@ ARRAY=AAA
 QC="QQQQ"
 
 # set the right observatory (environmental variables)
-source $EVNDISPSYS/setObservatory.sh CTA
+if [ ! -n "$EVNDISP_APPTAINER" ]; then
+    source "${EVNDISPSYS}"/setObservatory.sh CTA
+fi
 
-# output data files are written to this directory
-mkdir -p $ODIR
+# temporary (scratch) directory
+if [[ -n $TMPDIR ]]; then
+  TEMPDIR=$TMPDIR/$RUN
+else
+  TEMPDIR="$CTA_USER_DATA_DIR/TMPDIR"
+fi
+echo "Scratch dir: $TEMPDIR"
+mkdir -p "$TEMPDIR"
 
+# output data files are written to this directory
+# (note that $OUTPUTDIR is pointing inside the
+# apptainer to a mounted directory, while $ODIR
+# is not changed)
+OUTPUTDIR="${ODIR}"
+mkdir -p $OUTPUTDIR
 # delete old log files
-rm -f $ODIR/${BDT}-${TTYPE}.training.log
+rm -f $OUTPUTDIR/${BDT}-${TTYPE}.training.log
 # delete old training files
-rm -f $ODIR/*${TTYPE}*
+rm -f $OUTPUTDIR/*${TTYPE}*
+
+# data data directory to LIST
+cp -v "$TLIST" "$TEMPDIR"
+
+if [ -n "$EVNDISP_APPTAINER" ]; then
+    APPTAINER_MOUNT=" --bind ${OUTPUTDIR}:/eventdisplay_datadir/output/ "
+    APPTAINER_MOUNT+=" --bind ${DDIR}:/eventdisplay_datadir/data/ "
+    APPTAINER_MOUNT+=" --bind ${TEMPDIR}:/eventdisplay_datadir/tmp/ "
+    echo "APPTAINER MOUNT: ${APPTAINER_MOUNT}"
+    APPTAINER_ENV="--env OUTPUTDIR=/eventdisplay_datadir/data/,TEMPDIR=/eventdisplay_datadir/tmp/"
+    EVNDISPSYS="${EVNDISPSYS/--cleanenv/--cleanenv $APPTAINER_ENV $APPTAINER_MOUNT}"
+    echo "APPTAINER SYS: $EVNDISPSYS"
+    OUTPUTDIR="/eventdisplay_datadir/output/"
+    DDIR="/eventdisplay_datadir/data"
+    sed -i "s|^|${DDIR}/|" "$TEMPDIR/$(basename $TLIST)"
+    TEMPDIR="/eventdisplay_datadir/tmp/"
+else
+    sed -i "s|^|${DDIR}/|" "$TEMPDIR/$(basename $TLIST)"
+fi
 
 # array layout file
 
@@ -51,15 +82,15 @@ elif [[ $DSET == *"prod5"* ]]; then
 elif [[ $DSET == *"prod6"* ]]; then
     ADIR=$CTA_EVNDISP_AUX_DIR/DetectorGeometry/CTA.prod6${DARR}.lis
 else
-    echo "Unknown data set: $DSET"
-    echo "exiting..."
+    echo "Unknown data set: $DSET, exiting"
     exit
 fi
 
-#########################################
+echo $OUTPUTDIR
+
 # train TMVA
-$EVNDISPSYS/bin/trainTMVAforAngularReconstruction $TLIST \
-                                                  $ODIR \
+$EVNDISPSYS/bin/trainTMVAforAngularReconstruction "$TEMPDIR/$(basename $TLIST)" \
+                                                  "$OUTPUTDIR" \
                                                   0.8 \
                                                   ${RECID} \
                                                   ${TTYPE} \
@@ -68,19 +99,16 @@ $EVNDISPSYS/bin/trainTMVAforAngularReconstruction $TLIST \
                                                   ${ADIR} \
                                                   "" \
                                                   ${QC} \
-                                                  0 > $ODIR/${BDT}-${TTYPE}.training.log 2>&1
-#########################################
+                                                  0 > # $ODIR/${BDT}-${TTYPE}.training.log 2>&1
+
 
-##############
 # cleanup
 # remove everything if telescope type is not found
 if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]]; then
     if grep -Fxq "Number of telescope types: 0" $ODIR/${BDT}-${TTYPE}.training.log
     then
          echo "No telescopes found of type ${TTYPE}"
          head -n 10 ${TLIST}
-    #     rm -f -v $ODIR/${BDT}"_"${TTYPE}.root
-    #     rm -f $ODIR/${BDT}-${TTYPE}.training.log
     fi
 fi
 # move everything into root files
@@ -92,12 +120,6 @@ if [[ -e $ODIR/${BDT}_MLP_${TTYPE}.weights.xml ]]; then
     $EVNDISPSYS/bin/logFile dispXML-MLP-${TTYPE} $ODIR/${BDT}-${TTYPE}.disptmva.root $ODIR/${BDT}_MLP_${TTYPE}.weights.xml
     rm -f $ODIR/${BDT}_MLP_${TTYPE}.weights.xml
 fi
-if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]]; then
+if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]] && [[ -e $ODIR/${BDT}-${TTYPE}.disptmva.root ]]; then
     $EVNDISPSYS/bin/logFile dispLog-${TTYPE} $ODIR/${BDT}-${TTYPE}.disptmva.root $ODIR/${BDT}-${TTYPE}.training.log
-#    rm -f $ODIR/${BDT}-${TTYPE}.training.log
 fi
-
-#rm -f $ODIR/${BDT}_${TTYPE}.root
-#rm -f $ODIR/${BDT}_${TTYPE}.tmva.root
-
-exit
diff --git a/analysis/CTA.DISPTRAINING.sub_analyse.sh b/analysis/CTA.DISPTRAINING.sub_analyse.sh
@@ -34,7 +34,7 @@ then
    echo "  <TMVA parameters>  file name of list of TMVA parameter file"
    echo "  <scaling>          layout scaling (e.g. 5); give 99 to ignore scaling"
    echo
-   echo "  (note 1: hardwired telescope types in this script)" 
+   echo "  (note 1: hardwired telescope types in this script)"
    echo "  (note 2: disp core training switched off)"
    echo
    exit
@@ -62,8 +62,8 @@ if [ -n $8 ]
 then
    QSUBOPT="$8"
 fi
-QSUBOPT=${QSUBOPT//_X_/ } 
-QSUBOPT=${QSUBOPT//_M_/-} 
+QSUBOPT=${QSUBOPT//_X_/ }
+QSUBOPT=${QSUBOPT//_M_/-}
 
 #########################################
 # TMVA options
@@ -153,7 +153,7 @@ else
 fi
 
 #########################################
-# 
+#
 #########################################
 for MLP in "${MLPLIST[@]}"
 do
@@ -190,7 +190,8 @@ do
                 ####################
                 # input file list
                 rm -f $SHELLDIR/tempList.list
-                find $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${EVNDISP}/gamma_cone/ -name "*[_,.]${MCAZ}*.root" > $SHELLDIR/tempList.list
+                DATADIR="$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${EVNDISP}/gamma_cone/"
+                find ${DATADIR} -name "*[_,.]${MCAZ}*.root"  -printf "%f\n" > $SHELLDIR/tempList.list
                 NFIL=`wc -l $SHELLDIR/tempList.list | awk '{print $1}'`
                 echo "Total number of files available: $NFIL"
                 # only use NN% of all evndisp files for training
@@ -226,7 +227,8 @@ do
                 FNAM="$SHELLDIR/EDISP-$ARRAY-$SCALING-$MCAZ-$TELTYPE-$MLP-$NSTEP"
                 cp $FSCRIPT.sh $FNAM.sh
 
-                  sed -i -e "s|OFILE|$TDIR|" \
+                  sed -i -e "s|OODIR|$TDIR|" \
+                         -e "s|DDDIR|$DATADIR|" \
                          -e "s|TELTYPE|$TELTYPE|" \
                          -e "s|MLPTYPE|$MLP|" \
                          -e "s|RECONSTRUCTIONID|$RECID|" \

diff --git a/analysis/CTA.EFFAREA.qsub_analyse_list.sh b/analysis/CTA.EFFAREA.qsub_analyse_list.sh
@@ -2,16 +2,16 @@
 #
 # calculate effective areas and instrument response functions for CTA
 #
-#
-#
-##############################################################################
 
 echo
 echo "calculating effective areas for CTA: create run scripts"
 echo "------------------------------------------------"
 echo
 
-source ${EVNDISPSYS}/setObservatory.sh CTA
+# set the right observatory (environmental variables)
+if [ ! -n "$EVNDISP_APPTAINER" ]; then
+    source "${EVNDISPSYS}"/setObservatory.sh CTA
+fi
 
 ######################################################################
 # input variables
@@ -45,7 +45,7 @@ echo "PROCESSING PARTICLE $PART"
 # read analysis values from parameter file
 if [ ! -e $ANAPAR ]
 then
-  echo "error: analysis parameter file not found: $ANAPAR" 
+  echo "error: analysis parameter file not found: $ANAPAR"
   exit
 fi
 cp -f $ANAPAR $TMPDIR
@@ -54,7 +54,7 @@ ANAPAR="${TMPDIR}/${ANAPARF}"
 # check again than runparameter file is available
 if [ ! -e $ANAPAR ]
 then
-  echo "error: analysis parameter file not found in tmp directory: $ANAPAR" 
+  echo "error: analysis parameter file not found in tmp directory: $ANAPAR"
   exit
 fi
 echo "reading analysis parameter from $ANAPAR"
@@ -74,7 +74,7 @@ then
 fi
 
 NIMAGESMIN=`grep NIMAGESMIN $ANAPAR | awk {'print $2'}`
-# get telescope type dependent cuts 
+# get telescope type dependent cuts
 NCUTLST=`grep NLST $ANAPAR | awk {'print $2'}`
 NCUTMST=`grep NMST $ANAPAR | awk {'print $2'}`
 NCUTSST=`grep NSST $ANAPAR | awk {'print $2'}`
@@ -100,7 +100,7 @@ else
 fi
 THETA2MINENEERGY=`grep THETA2MINENEERGY $ANAPAR | awk {'print $2'}`
 
-for T in LST MST SST SCMST 
+for T in LST MST SST SCMST
 do
     NCUT="NCUT${T}"
     if [ -z "${!NCUT}" ]
@@ -140,7 +140,7 @@ echo "  Analysis parameters: $NIMAGESMIN $ANADIR $TMVACUT $EFFAREABASEDIR $OBSTI
 
 if [ -z "$ANADIR" ] || [ -z "$NIMAGESMIN" ] || [ -z "$TMVACUT" ] || [ -z "$EFFAREABASEDIR" ] || [ -z "$OBSTIME" ]
 then
-  echo "error: analysis parameter file not correct: $ANAPAR" 
+  echo "error: analysis parameter file not correct: $ANAPAR"
   echo " one variable missing"
   exit
 fi
@@ -259,18 +259,18 @@ then
       THETA2MIN=( 0. )
       THETA2MAX=( 1. )
       OFFMEA=( 0.5 )
-   fi   
+   fi
    ISOTROPY="1"
    DIRECTIONCUT="0"
-fi 
+fi
 if [ $PART = "proton" ] || [ $PART = "proton_onSource" ]
 then
    if [[ ${DSET:0:2} == "GR" ]]
    then
        MSCFILE=$DDIR/proton*"deg$MCAZ"*mscw.root
    else
        MSCFILE=$DDIR/proton*."$ARRAY"_ID"$RECID$MCAZ"*.mscw.root
-   fi    
+   fi
    if [ $ARRAY = "V5" ]
    then
       MSCFILE=$DDIR/proton."$ARRAY"_ID"$RECID$MCAZ".mscw.root
@@ -279,7 +279,7 @@ then
    OFFMIN=( 0. )
    OFFMAX=( 100000. )
 # NOTE: this is theta and not theta2
-   if [ $PART = "proton" ] 
+   if [ $PART = "proton" ]
    then
       OFIL=proton."$ARRAY"_ID"$RECID".eff
        if [ $BFINEBINNING = "TRUE" ]
@@ -389,8 +389,8 @@ do
           fi
     ###############################################################################
     # create cut file
-          iCBFILE=`basename $CFIL`      
-          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] 
+          iCBFILE=`basename $CFIL`
+          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ]
           then
               CFILP="${CFIL}.gamma.dat"
           else
@@ -407,14 +407,14 @@ do
           cp -f $CFILP $iCFIL
 
     # wobble offset
-          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] 
+          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ]
           then
              WOBBLEOFFSET=${OFFMEA[$i]}
           else
              WOBBLEOFFSET=${OFFMEA[$j]}
           fi
     # angular resolution file
-          if [ $PART = "gamma_onSource" ] 
+          if [ $PART = "gamma_onSource" ]
           then
              ANGRESFILE=${PRODBASEDIR}/EffectiveAreas/${EFFMCAZDIR}/AngularResolution/gamma_onSource."$ARRAY"_ID"$RECID".eff-0.root
           else
@@ -522,9 +522,9 @@ do
          echo "* ENERGYSPECTRUMINDEX  1 2.5 0.1" >> $MSCF
          echo "* ESPECTRUM_FOR_WEIGHTING $CTA_EVNDISP_AUX_DIR/AstroData/TeV_data/EnergySpectrum_literatureValues_CR.dat 0" >> $MSCF
          if  [ $GETXOFFYOFFAFTERCUTS = "yes" ]
-         then	
+         then
              echo "* GETXOFFYOFFAFTERCUTS 1" >> $MSCF
-         fi    
+         fi
 
       fi
       if [ $PART = "electron" ] || [ $PART = "electron_onSource" ]
@@ -587,7 +587,7 @@ do
   # run effective area code
       ${EVNDISPSYS}/bin/makeEffectiveArea $MSCF $OFIX.root > $OLOG.log
 
-      # cross check if run was successfull
+      # cross check if run was successful
       # (expect simply > 800k)
       DS=$(du -k $OFIX.root | cut -f 1)
       if [[ ${DS} -le $minimumsize ]]; then
@@ -626,6 +626,3 @@ do
 done
 
 exit
-
-
-exit