-
Notifications
You must be signed in to change notification settings - Fork 54
Jobscript tips and examples
K Clough edited this page May 2, 2024
·
10 revisions
In general you should start with an example hybrid MPI/OpenMP jobscript from your cluster documentation, and adapt it for your GRChombo run.
However, some example jobs for systems we use are collected here - to be copied at your own risk!
#!/bin/bash --login
#!
#! Example SLURM job script for Archer2
#! Last updated: Thu Jul 13 2023
#!
#!##############################################################################
#! Change what you need
#!##############################################################################
#! Job Name and Files (also --job-name)
#SBATCH -J JobName
#! Singleton? Only run one job with this name at a time
#SBATCH --dependency=singleton
#! Wall clock limit:
#SBATCH --time=12:00:00
#! Number of nodes and MPI tasks per node:
#SBATCH --nodes=4
#! There are 128 physical cores per node, so
#! the product of these numbers should be 128
#SBATCH --ntasks-per-node=64
#SBATCH --cpus-per-task=2
#! Notification and type (choose NONE, BEGIN, END, FAIL, REQUEUE, ALL)
#! NOT WORKING ON ARCHER2
#SBATCH --mail-type=ALL
#SBATCH [email protected]
#!##############################################################################
#! Probably fixed:
#!##############################################################################
#! Output and error (also --output, --error):
#SBATCH -o output.%J.txt
#SBATCH -e error.%J.txt
#! Initial working directory (also --chdir):
#! Archer requires this to be inside /work/
#SBATCH -D ./
#! Partition to use (only 'standard')
#SBATCH --partition=standard
#SBATCH --qos=standard
#! Setup of execution environment
#SBATCH --export=NONE
#SBATCH --get-user-env
#SBATCH --account=e775
#! let OMP know how many threads
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
export OMP_PLACES=cores #! for archer2
export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK #! necessary for hybrid jobs
#!##############################################################################
#! Load modules
#!##############################################################################
#! Nothing extra needed as flag '#SBATCH --get-user-env' above
#! inherits the same modules loaded from '.bashrc' at login
#! but anyway adding the relevant modules
module load PrgEnv-cray # default cray environment
module load cray-hdf5-parallel # parallel HDF5 libraries
module load petsc # default petsc library
# use large pages in normal Chombo
# documentation here: https://docs.archer2.ac.uk/user-guide/scheduler/
module load craype-hugepages2M
# for whoever uses TwoPunctures
module load gsl
#! Use Chombo Timers (produces 'LB.txt' and the 'time.table.*' files)
# export CH_TIMER=TRUE
#!##############################################################################
#! TO CHANGE
#!##############################################################################
EXEC="program"
PARAMS="params.txt"
OPTIONS=""
WORKDIR="$SLURM_SUBMIT_DIR"
CMD_BASE="time srun --hint=nomultithread --distribution=block:block"
#!##############################################################################
#! Print some useful job info
#!##############################################################################
cd $WORKDIR
echo -e "\nRunning on master node: `hostname`"
echo "Current directory: `pwd`"
echo ID $SLURM_JOB_ID
echo PROCID $SLURM_PROCID
echo NAME $SLURM_JOB_NAME
echo PARTITION $SLURM_JOB_PARTITION
echo
echo NODES $SLURM_JOB_NUM_NODES
echo NODELIST $SLURM_JOB_NODELIST
echo TASKS $SLURM_NTASKS
echo TASKS_PER_NODE $SLURM_NTASKS_PER_NODE
echo CPUS_PER_TASK $SLURM_CPUS_PER_TASK
echo MEM_PER_NODE $SLURM_MEM_PER_NODE
echo
module list
#!##############################################################################
#! RUN
#!##############################################################################
#! uncomment to delete old checkpoints, update restart file and save pout files
# source /work/pr1u1753/pr1u1753/$USER/restart_functions.sh
# export FILES_TO_KEEP=2
# update_before_running params.txt $FILES_TO_KEEP use_plot
#! Run the program:
CMD="$CMD_BASE $EXEC $PARAMS $OPTIONS"
echo "Time: `date`" # print start time
echo -e "\nExecuting command:\n==================\n$CMD\n"
eval $CMD
echo -e "Time: `date`\n" #! print finish time
exit
#!/bin/bash
#! Name of the job:
#SBATCH -J job_name
#! Which project should be charged:
#SBATCH -A project_name (in the form dirac-dpxxx-cpu or gpu)
#! Which partition should be used:
#SBATCH -p icelake-himem
#! How many whole nodes should be allocated?
#SBATCH --nodes=6
#! How many tasks per node
#SBATCH --ntasks-per-node=38
#! How many cores per task
#SBATCH -c 2
#! How much wallclock time will be required?
#SBATCH --time=36:00:00
#! What types of email messages do you wish to receive?
#SBATCH --mail-type=all
#SBATCH --error=err.out
#! Uncomment this to prevent the job from being requeued (e.g. if
#! interrupted by node failure or system downtime):
##SBATCH --no-requeue
#! sbatch directives end here (put any additional directives above this line)
#! Notes:
#! Charging is determined by cpu number*walltime.
#! Number of nodes and tasks per node allocated by SLURM (do not change):
numnodes=$SLURM_JOB_NUM_NODES
numtasks=$SLURM_NTASKS
mpi_tasks_per_node=$SLURM_NTASKS_PER_NODE
#! ############################################################
#! Modify the settings below to specify the application's environment, location
#! and launch method:
#! Optionally modify the environment seen by the application
#! (note that SLURM reproduces the environment at submission irrespective of ~/.bashrc):
. /etc/profile.d/modules.sh # Leave this line (enables the module command)
module purge # Removes all modules still loaded
module load rhel8/default-icl # REQUIRED - loads the basic environment
#! Insert additional module load commands after this line if needed:
module load hdf5/mpi/intel/2019.3/1.10.5
#! Full path to application executable:
application="path_to_executable"
#! Run options for the application:
options="params.txt"
#! Work directory (i.e. where the job will run):
workdir="$SLURM_SUBMIT_DIR" # The value of SLURM_SUBMIT_DIR sets workdir to the directory
# in which sbatch is run.
#! Are you using OpenMP (NB this is unrelated to OpenMPI)? If so increase this
#! safe value to no more than 56:
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
export OMP_PROC_BIND=true
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mkl/lib/intel64_lin/"
#! Number of MPI tasks to be started by the application per node and in total (do not change):
np=$[${numnodes}*${mpi_tasks_per_node}]
#! The following variables define a sensible pinning strategy for Intel MPI tasks -
#! this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
#export I_MPI_PIN_DOMAIN=omp:compact # Domains are $OMP_NUM_THREADS cores in size
#export I_MPI_PIN_ORDER=compact # Adjacent domains have minimal sharing of caches/sockets
#! Notes:
#! 1. These variables influence Intel MPI only.
#! 2. Domains are non-overlapping sets of cores which map 1-1 to MPI tasks.
#! 3. I_MPI_PIN_PROCESSOR_LIST is ignored if I_MPI_PIN_DOMAIN is set.
#! 4. If MPI tasks perform better when sharing caches/sockets, try I_MPI_PIN_ORDER=compact.
#! Uncomment one choice for CMD below (add mpirun/mpiexec options if necessary):
#! Choose this for a MPI code (possibly using OpenMP) using Intel MPI.
CMD="srun -K --distribution=block:block $application $options"
#CMD="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
###############################################################
### You should not have to change anything below this line ####
###############################################################
cd $workdir
echo -e "Changed directory to `pwd`.\n"
JOBID=$SLURM_JOB_ID
echo -e "JobID: $JOBID\n======"
echo "Time: `date`"
echo "Running on master node: `hostname`"
echo "Current directory: `pwd`"
if [ "$SLURM_JOB_NODELIST" ]; then
#! Create a machine file:
export NODEFILE=`generate_pbs_nodefile`
cat $NODEFILE | uniq > machine.file.$JOBID
echo -e "\nNodes allocated:\n================"
echo `cat machine.file.$JOBID | sed -e 's/\..*$//g'`
fi
module list
echo -e "\nnumtasks=$numtasks, numnodes=$numnodes, mpi_tasks_per_node=$mpi_tasks_per_node (OMP_NUM_THREADS=$OMP_NUM_THREADS)"
echo -e "\nExecuting command:\n==================\n$CMD\n"
eval $CMD
#!/bin/bash -l
#SBATCH --job-name=job_name
#SBATCH -o output%j
#SBATCH -e error%j
#SBATCH -p slurm
#SBATCH -A project_name
#SBATCH -p slurm
# Number of nodes
#SBATCH --nodes=4
# Dial3 has 128 cores per node so product of these = 128
#SBATCH --tasks-per-node=64
#SBATCH --cpus-per-task=2
#SBATCH --time=36:00:00
#SBATCH --exclusive
# turn on all mail notification
#SBATCH --mail-type=ALL
#SBATCH --mail-user=your_email
# Remove all previously loaded modules.
module purge
module load PrgEnv-cray/8.0.0
module load cray-hdf5
#=====================================
# Only needed for new partition
#=====================================
module load ucx/1.12.0
export UCX_SHM_DEVICES=memory
export UCX_TLS=rc,xpmem,self
#=====================================
# Only needed if using libfabrics
#=====================================
#module use /opt/cray/modulefiles
#module load libfabric/1.10.2pre1
#export FI_PROVIDER=verbs
#export LD_LIBRARY_PATH="/home/dc-evst1/gsl/lib":$LD_LIBRARY_PATH
export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
echo "Running on $SLURM_JOB_NUM_NODES nodes, with $SLURM_TASKS_PER_NODE ranks, and ${OM_NUM_THREADS} threads per Rank"
#ldd ./your_executable (assuming in the same directory path as your JOBSCRIPT)
srun --distribution=block:block ./your_executable params.txt
#!/bin/bash -l
#! How many nodes
#SBATCH --nodes 6
### NB cosma7 has 28 cores per node so product of these = 28
#SBATCH --ntasks-per-node=14 ## Total tasks will be this * number of nodes
#SBATCH --cpus-per-task=2 ## Equal to number of OMP threads per task (set below)
#! How many CPUs per tasks should be allocated (for OpenMP threading)
#SBATCH --cpus-per-task=2
#SBATCH -J job_name
#SBATCH -o output.out
#SBATCH -e error.err
#SBATCH -p cosma7
#SBATCH -A project_name
#SBATCH --exclusive
#SBATCH -t 36:00:00
#SBATCH --mail-type=ALL # notifications for job done & fail
#SBATCH --mail-user=your_email
module purge
module load intel_comp/2019
module load intel_mpi/2019
module load parallel_hdf5/1.10.3
#!Print info
module list
pwd
date
#! Are you using OpenMP?
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
#! Number of nodes and tasks per node allocated by SLURM (do not change):
mpi_tasks_per_node=$(echo "$SLURM_TASKS_PER_NODE" | sed -e 's/^\([0-9][0-9]*\).*$/\1/')
#! Number of nodes and tasks per node allocated by SLURM (do not change):
numnodes=$SLURM_JOB_NUM_NODES
numtasks=$SLURM_NTASKS
#! Number of MPI tasks to be started by the application per node and in total (do not change):
np=$[${numnodes}*${mpi_tasks_per_node}]
#! Full path to application executable:
application="Path_to_executable"
#! Run options for the application:
options="params.txt"
#! Work directory (i.e. where the job will run):
workdir="$SLURM_SUBMIT_DIR"
# Run the program
mpirun -ppn $mpi_tasks_per_node -np $SLURM_NTASKS $application $options
#!/bin/bash -l
# number of nodes
#SBATCH --nodes 1
#! How many tasks
#SBATCH --ntasks-per-node=64
#! How many CPUs per tasks should be allocated (for OpenMP threading)
#! The product of this and ntasks-per-node should be 128
#SBATCH --cpus-per-task=2
#SBATCH -J job_name
#SBATCH -o output.out
#SBATCH -e error.err
#SBATCH -p cosma8
#SBATCH -A project_name
#SBATCH --exclusive
#SBATCH -t 36:00:00
#SBATCH --mail-type=ALL # notifications for job done & fail
#SBATCH --mail-user=your_email
module purge
module load gnu_comp/7.3.0
module load intel_comp/2022.1.2
module load compiler/2022.0.2 mkl/2022.0.2 mpi/2021.5.1
module load parallel_hdf5/1.12.0
#!Print info
module list
pwd
date
#! Are you using OpenMP?
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
#! Number of nodes and tasks per node allocated by SLURM (do not change):
mpi_tasks_per_node=$(echo "$SLURM_TASKS_PER_NODE" | sed -e 's/^\([0-9][0-9]*\).*$/\1/')
#! Number of nodes and tasks per node allocated by SLURM (do not change):
numnodes=$SLURM_JOB_NUM_NODES
numtasks=$SLURM_NTASKS
#! Number of MPI tasks to be started by the application per node and in total (do not change):
np=$[${numnodes}*${mpi_tasks_per_node}]
#! Full path to application executable:
application="path_to_executable"
#! Run options for the application:
options="params.txt"
#! Work directory (i.e. where the job will run):
workdir="$SLURM_SUBMIT_DIR"
# Run the program
mpirun -ppn $mpi_tasks_per_node -np $SLURM_NTASKS $application $options
Copyright GRChombo 2018. Contact us for further details.