diff --git a/LOG/SLURM/.gitignore b/LOG/SLURM/.gitignore new file mode 100644 index 0000000..1c1d568 --- /dev/null +++ b/LOG/SLURM/.gitignore @@ -0,0 +1,3 @@ +*.out +*.err +*.log diff --git a/scripts/submit_csv.sh b/scripts/submit_csv.sh index 39b4b86..b4e72fa 100755 --- a/scripts/submit_csv.sh +++ b/scripts/submit_csv.sh @@ -36,6 +36,11 @@ else CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET}) fi +# number of jobs (zero-based) +NUM_JOBS=$(wc -l ${CSV_FILE} | cut -f1 -d\ ) +let NUM_JOBS=${NUM_JOBS}-1 +echo ${NUM_JOBS} + # create command line EXECUTABLE="$(dirname $0)/run.sh" ARGUMENTS="EVGEN/\$(file) \$(ext) \$(nevents) \$(ichunk)" @@ -72,11 +77,28 @@ sed " " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE} # submit job -condor_submit -verbose -file ${SUBMIT_FILE} +if [[ ${TEMPLATE} =~ slurm ]] ; then -# create log dir -if [ $? -eq 0 ] ; then - for i in `condor_q --batch | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do - mkdir -p LOG/CONDOR/osg_$i/ + # slurm + max_array_size=2000 + while [ ${NUM_JOBS} -ge 0 ] ; do + let taskN=${NUM_JOBS} + let task0=${NUM_JOBS}-${max_array_size}+1 + if [ ${task0} -lt 0 ] ; then task0=0 ; fi + sbatch --array=${task0}-${taskN} ${SUBMIT_FILE} + let NUM_JOBS=${NUM_JOBS}-${max_array_size} done + +else + + # condor + condor_submit -verbose -file ${SUBMIT_FILE} + + # create log dir + if [ $? -eq 0 ] ; then + for i in `condor_q --batch | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do + mkdir -p LOG/CONDOR/osg_$i/ + done + fi + fi diff --git a/templates/slurm_csv.sh.in b/templates/slurm_csv.sh.in new file mode 100644 index 0000000..2d14f21 --- /dev/null +++ b/templates/slurm_csv.sh.in @@ -0,0 +1,7 @@ +export COPYRECO=%COPYRECO% +export COPYFULL=%COPYFULL% +export COPYLOG=%COPYLOG% +export DETECTOR_VERSION=%DETECTOR_VERSION% +export DETECTOR_CONFIG=%DETECTOR_CONFIG% +export EBEAM=%EBEAM% +export PBEAM=%PBEAM% diff --git a/templates/slurm_csv.submit.in b/templates/slurm_csv.submit.in new file mode 100644 index 0000000..31ed738 --- /dev/null +++ b/templates/slurm_csv.submit.in @@ -0,0 +1,38 @@ +#!/bin/bash +#SBATCH --job-name=%DETECTOR_VERSION%/%DETECTOR_CONFIG%/%CSV_FILE% +#SBATCH --error=LOG/SLURM/grex_%A/grex_%A_%4a.err +#SBATCH --output=LOG/SLURM/grex_%A/grex_%A_%4a.out +#SBATCH --mail-type=FAIL,TIME_LIMIT,TIME_LIMIT_90 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=3G +#SBATCH --time=24:00:00 + +# Load singularity module +module load singularity + +# Get CSV_FILE entry $a +mapfile array < %CSV_FILE% +IFS=, read file ext nevents ichunk <<< ${array[${SLURM_ARRAY_TASK_ID}]} + +# Start singularity instance +SingularityImage="/cvmfs/singularity.opensciencegrid.org/eicweb/eic_xl:%JUG_XL_TAG%" +instance=${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID} +singularity instance start --bind /cvmfs --bind $(realpath $PWD) ${SingularityImage} ${instance} + +# Create aliases for condor syntax +set -a # export the following funcs +file() { echo $file; } +ext() { echo $ext; } +nevents() { echo $nevents; } +ichunk() { echo $ichunk; } +set +a # stop exporting + +# Run command +singularity exec instance://${instance} /bin/bash << EOF +cd $(realpath $PWD) +source $(basename $0 .submit).sh +%EXECUTABLE% %ARGUMENTS% +EOF + +# Stop singularity instance +singularity instance stop ${instance} diff --git a/test.submit b/test.submit deleted file mode 100644 index 0ddd09c..0000000 --- a/test.submit +++ /dev/null @@ -1,13 +0,0 @@ -universe = vanilla -executable = test.sh - -output = test.out -error = test.err -log = test.log - -should_transfer_files = Yes -when_to_transfer_output = ON_EXIT - -#+ProjectName = "EIC" - -queue