feat: merge slurm support

LOG/SLURM/.gitignore

@@ -0,0 +1,3 @@
+*.out
+*.err
+*.log

scripts/submit_csv.sh

@@ -36,6 +36,11 @@ else
   CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET})
 fi
 
+# number of jobs (zero-based)
+NUM_JOBS=$(wc -l ${CSV_FILE} | cut -f1 -d\ )
+let NUM_JOBS=${NUM_JOBS}-1
+echo ${NUM_JOBS}
+
 # create command line
 EXECUTABLE="$(dirname $0)/run.sh"
 ARGUMENTS="EVGEN/\$(file) \$(ext) \$(nevents) \$(ichunk)"
@@ -72,11 +77,28 @@ sed "
 " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE}
 
 # submit job
-condor_submit -verbose -file ${SUBMIT_FILE}
+if [[ ${TEMPLATE} =~ slurm ]] ; then
 
-# create log dir
-if [ $? -eq 0 ] ; then
-  for i in `condor_q --batch | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do
-    mkdir -p LOG/CONDOR/osg_$i/
+  # slurm
+  max_array_size=2000
+  while [ ${NUM_JOBS} -ge 0 ] ; do
+    let taskN=${NUM_JOBS}
+    let task0=${NUM_JOBS}-${max_array_size}+1
+    if [ ${task0} -lt 0 ] ; then task0=0 ; fi
+    echo sbatch --array=${task0}-${taskN} ${SUBMIT_FILE}
+    let NUM_JOBS=${NUM_JOBS}-${max_array_size}
   done
+
+else
+
+  # condor
+  condor_submit -verbose -file ${SUBMIT_FILE}
+
+  # create log dir
+  if [ $? -eq 0 ] ; then
+    for i in `condor_q --batch | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do
+      mkdir -p LOG/CONDOR/osg_$i/
+    done
+  fi
+
 fi

templates/slurm_csv.sh.in

@@ -0,0 +1,7 @@
+export COPYRECO=%COPYRECO%
+export COPYFULL=%COPYFULL%
+export COPYLOG=%COPYLOG%
+export DETECTOR_VERSION=%DETECTOR_VERSION%
+export DETECTOR_CONFIG=%DETECTOR_CONFIG%
+export EBEAM=%EBEAM%
+export PBEAM=%PBEAM%

templates/slurm_csv.submit.in

@@ -0,0 +1,38 @@
+#!/bin/bash
+#SBATCH --job-name=%DETECTOR_VERSION%/%DETECTOR_CONFIG%/%CSV_FILE%
+#SBATCH --error=LOG/SLURM/grex_%A/grex_%A_%4a.err
+#SBATCH --output=LOG/SLURM/grex_%A/grex_%A_%4a.out
+#SBATCH --mail-type=FAIL,TIME_LIMIT,TIME_LIMIT_90
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=3G
+#SBATCH --time=24:00:00
+
+# Load singularity module
+module load singularity
+
+# Get CSV_FILE entry $a
+mapfile array < %CSV_FILE%
+IFS=, read file ext nevents ichunk <<< ${array[${SLURM_ARRAY_TASK_ID}]}
+
+# Start singularity instance
+SingularityImage="/cvmfs/singularity.opensciencegrid.org/eicweb/eic_xl:%JUG_XL_TAG%"
+instance=${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}
+singularity instance start --bind /cvmfs --bind $(realpath $PWD) ${SingularityImage} ${instance}
+
+# Create aliases for condor syntax
+set -a # export the following funcs
+file()    { echo $file; }
+ext()     { echo $ext; }
+nevents() { echo $nevents; }
+ichunk()  { echo $ichunk; }
+set +a # stop exporting
+
+# Run command
+singularity exec instance://${instance} /bin/bash << EOF
+cd $(realpath $PWD)
+source $(basename $0 .submit).sh
+%EXECUTABLE% %ARGUMENTS%
+EOF
+
+# Stop singularity instance
+singularity instance stop ${instance}