geoschem · yantosca · Aug 29, 2024 · Aug 28, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Removed unused `VDIFFAR` routine from `vdiff_mod.F90`.
 - Update MW for CH4 and OH in `global_ch4_mod.F90`.
 - Do not convert from kg/kg to mol/mol before passing State_Chm to PBL mixing in `vdiff_mod.F90`.
+- Updated GC-Classic and GCHP run scripts and environment files for NASA discover cluster
 
 ### Fixed
 - Simplified SOA representations and fixed related AOD and TotalOA/OC calculations in benchmark.

diff --git a/run/GCClassic/runScriptSamples/operational_examples/nasa_discover/gcc.discover.build b/run/GCClassic/runScriptSamples/operational_examples/nasa_discover/gcc.discover.build
diff --git a/run/GCClassic/runScriptSamples/operational_examples/nasa_discover/gcc.discover.run b/run/GCClassic/runScriptSamples/operational_examples/nasa_discover/gcc.discover.run
@@ -1,11 +1,13 @@
 #!/bin/bash
 
 # For discover
-#SBATCH --time 00:45:00
-#SBATCH --nodes=1 --ntasks-per-node=24
-#SBATCH --job-name=gchp_build
-#SBATCH --constraint=sky
-#SBATCH --account=s____
+#SBATCH --time 1:00:00
+#SBATCH --nodes=1 --ntasks-per-node=48
+#SBATCH --mem 4G         # Use 10G for 2x2.5
+#SBATCH --job-name=gcclassic.run
+#SBATCH --constraint=mil # Edit as needed
+#SBATCH --account=s___   # Fill in
+##SBATCH --qos=__        # Fill in and uncomment if using
 
-source gcc.discover.env
+source gcc.env  # Edit, or create gcc.env symlink to env file
 ./gcclassic > gcc.log
diff --git a/...l_examples/nasa_discover/gcc.discover.env → ...s/nasa_discover/gcc.spack-stack-1.6.0.env b/...l_examples/nasa_discover/gcc.discover.env → ...s/nasa_discover/gcc.spack-stack-1.6.0.env
@@ -1,14 +1,15 @@
 #!/bin/bash
 
+# NOTE: not usable on milan nodes
+
 module purge
 module use /discover/swdev/jcsda/spack-stack/modulefiles
-module load miniconda/3.9.7
-module use /discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.1/install/modulefiles/Core
+module use /discover/swdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
 module load stack-intel/2022.0.1
 module load stack-intel-oneapi-mpi/2021.5.0
 module load cmake/3.23.1
 module load git/2.30.2
-module load netcdf-fortran/4.5.4
+module load netcdf-fortran/4.6.1
 
 umask 022
 

diff --git a/...c/runScriptSamples/operational_examples/nasa_discover/gcc.spack-stack-1.7.0-gnu.milan.env b/...c/runScriptSamples/operational_examples/nasa_discover/gcc.spack-stack-1.7.0-gnu.milan.env
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# NOTE: use on milan nodes, accessible with `ssh discover-mil`
+# See also https://spack-stack.readthedocs.io/en/latest/PreConfiguredSites.html#nasa-discover-scu17
+
+module purge
+
+module use /discover/swdev/gmao_SIteam/modulefiles-SLES15
+module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles
+module load ecflow/5.11.4
+
+module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-gcc-12.3.0/install/modulefiles/Core
+module load stack-gcc/12.3.0
+module load stack-openmpi/4.1.6
+module load stack-python/3.10.13
+
+module load cmake/3.28.2
+module load git/2.42.0
+module load netcdf-fortran/4.6.1
+
+umask 022
+
+export CC=gcc
+export CXX=g++
+export FC=gfortran
+
+ulimit -l unlimited
+ulimit -u 50000
+ulimit -v unlimited
+ulimit -s unlimited
+
diff --git a/...runScriptSamples/operational_examples/nasa_discover/gcc.spack-stack-1.7.0-intel.milan.env b/...runScriptSamples/operational_examples/nasa_discover/gcc.spack-stack-1.7.0-intel.milan.env
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# NOTE: use this on milan nodes, accessible with `ssh discover-mil`
+# See also https://spack-stack.readthedocs.io/en/latest/PreConfiguredSites.html#nasa-discover-scu17
+
+module purge
+
+module use /discover/swdev/gmao_SIteam/modulefiles-SLES15
+module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles
+module load ecflow/5.11.4
+
+module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0/install/modulefiles/Core
+module load stack-intel/2021.10.0
+module load stack-intel-oneapi-mpi/2021.10.0
+module load stack-python/3.10.13
+
+module load cmake/3.28.2
+module load git/2.42.0
+module load netcdf-fortran/4.6.1
+
+umask 022
+
+export CC=icc
+export CXX=icpc
+export FC=ifort
+
+ulimit -l unlimited
+ulimit -u 50000
+ulimit -v unlimited
+ulimit -s unlimited
+
diff --git a/run/GCHP/runScriptSamples/operational_examples/nasa_discover/gchp.discover.build b/run/GCHP/runScriptSamples/operational_examples/nasa_discover/gchp.discover.build
@@ -1,18 +1,16 @@
 #!/bin/bash
 
 # For discover
-#SBATCH --time 00:45:00
-#SBATCH --nodes=1 --ntasks-per-node=12
-#SBATCH --job-name=gchp_build
-#SBATCH --constraint=sky
-#SBATCH --account=s1043
+#SBATCH --time 00:30:00
+#SBATCH --nodes=1 --ntasks-per-node=24
+#SBATCH --mem=6G
+#SBATCH --job-name=gchp.build
+#SBATCH --constraint=mil # Edit as needed
+#SBATCH --account=s____  # Fill in
+##SBATCH --qos=___       # Fill in and uncomment if using
 
-rundir=$(pwd -P)
-blddir=../build
-
-source ${rundir}/gchp.env
-cd ${blddir}
-cmake ${rundir}/CodeDir -DRUNDIR=${rundir}
-make -j > build.log
-make install >> build.log
-cd ${rundir}
+source gchp.env
+cd build
+cmake ../CodeDir -DRUNDIR=..
+make -j
+make install
diff --git a/run/GCHP/runScriptSamples/operational_examples/nasa_discover/gchp.discover.run b/run/GCHP/runScriptSamples/operational_examples/nasa_discover/gchp.discover.run
@@ -4,9 +4,11 @@
 #SBATCH --time 00:30:00
 #SBATCH --nodes=1 --ntasks-per-node=24
 #SBATCH --mem=60G
-#SBATCH --job-name=gchp_c30_run
-#SBATCH --constraint=sky
-#SBATCH --account=s1043
+#SBATCH --job-name=gchp.run
+#SBATCH --constraint=mil # Edit as needed
+#SBATCH --account=s___   # Fill in
+##SBATCH --qos=___       # Fill in and uncomment if using
+##SBATCH --exclusive     # Uncomment to reserve entire nodes
 
 # If a subsequent command fails, treat it as fatal (don't continue)
 set -e
@@ -22,7 +24,7 @@ ulimit -u 50000              # maxproc
 ulimit -v unlimited          # vmemoryuse
 ulimit -s unlimited          # stacksize
 
-source gchp.discover.env
+source gchp.env
 
 # Print loaded modules
 module list     

diff --git a/..._examples/nasa_discover/gchp.discover.env → .../nasa_discover/gchp.spack-stack-1.6.0.env b/..._examples/nasa_discover/gchp.discover.env → .../nasa_discover/gchp.spack-stack-1.6.0.env
@@ -2,13 +2,13 @@
 
 module purge
 module use /discover/swdev/jcsda/spack-stack/modulefiles
-module load miniconda/3.9.7
-module use /discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.1/install/modulefiles/Core
+#module load miniconda/3.9.7
+module use /discover/swdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
 module load stack-intel/2022.0.1
 module load stack-intel-oneapi-mpi/2021.5.0
 module load cmake/3.23.1
 module load git/2.30.2
-module load netcdf-fortran/4.5.4
+module load netcdf-fortran/4.6.1
 
 umask 022
 
@@ -23,7 +23,7 @@ export MPI_ROOT=${MPI_HOME}
 export ESMF_COMPILER=intel
 export ESMF_COMM=intelmpi
 export ESMF_DIR=/gpfsm/dnb06/projects/p113/ewl/GCHP/ESMF
-export ESMF_INSTALL_PREFIX=${ESMF_DIR}/INSTALL_discover-spack-stack
+export ESMF_INSTALL_PREFIX=${ESMF_DIR}/INSTALL_discover-spack-stack-1.6.0
 
 # For GCHP
 export ESMF_ROOT=${ESMF_INSTALL_PREFIX}

diff --git a/.../runScriptSamples/operational_examples/nasa_discover/gchp.spack-stack-1.7.0-gnu.milan.env b/.../runScriptSamples/operational_examples/nasa_discover/gchp.spack-stack-1.7.0-gnu.milan.env
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# NOTE: use on milan nodes, accessible with `ssh discover-mil`
+# See also https://spack-stack.readthedocs.io/en/latest/PreConfiguredSites.html#nasa-discover-scu17
+
+module purge
+
+module use /discover/swdev/gmao_SIteam/modulefiles-SLES15
+module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles
+module load ecflow/5.11.4
+
+module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-gcc-12.3.0/install/modulefiles/Core
+module load stack-gcc/12.3.0
+module load stack-openmpi/4.1.6
+module load stack-python/3.10.13
+
+module load cmake/3.28.2
+module load git/2.42.0
+module load netcdf-fortran/4.6.1
+
+umask 022
+
+export CC=gcc
+export CXX=g++
+export FC=gfortran
+
+# MPI
+export MPI_ROOT=${MPI_HOME}
+
+# For ESMF
+export ESMF_COMPILER=gfortran
+export ESMF_COMM=openmpi
+export ESMF_DIR=/gpfsm/dnb06/projects/p113/ewl/GCHP/ESMF
+export ESMF_INSTALL_PREFIX=${ESMF_DIR}/INSTALL_discover-spack-stack-1.7.0-gnu
+
+# For GCHP
+export ESMF_ROOT=${ESMF_INSTALL_PREFIX}
+
+ulimit -l unlimited
+ulimit -u 50000
+ulimit -v unlimited
+ulimit -s unlimited
+
diff --git a/...unScriptSamples/operational_examples/nasa_discover/gchp.spack-stack-1.7.0-intel.milan.env b/...unScriptSamples/operational_examples/nasa_discover/gchp.spack-stack-1.7.0-intel.milan.env
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# NOTE: use this with milan nodes, accessible with `ssh discover-mil`
+# See also https://spack-stack.readthedocs.io/en/latest/PreConfiguredSites.html#nasa-discover-scu17
+
+module purge
+
+module use /discover/swdev/gmao_SIteam/modulefiles-SLES15
+module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles
+module load ecflow/5.11.4
+
+module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0/install/modulefiles/Core
+module load stack-intel/2021.10.0
+module load stack-intel-oneapi-mpi/2021.10.0
+module load stack-python/3.10.13
+
+module load cmake/3.28.2
+module load git/2.42.0
+module load netcdf-fortran/4.6.1
+
+umask 022
+
+export CC=icc
+export CXX=icpc
+export FC=ifort
+
+# MPI
+export MPI_ROOT=${MPI_HOME}
+
+# For ESMF
+export ESMF_COMPILER=intel
+export ESMF_COMM=intelmpi
+export ESMF_DIR=/gpfsm/dnb06/projects/p113/ewl/GCHP/ESMF
+export ESMF_INSTALL_PREFIX=${ESMF_DIR}/INSTALL_discover-spack-stack-1.7.0-intel
+
+# For GCHP
+export ESMF_ROOT=${ESMF_INSTALL_PREFIX}
+
+ulimit -l unlimited
+ulimit -u 50000
+ulimit -v unlimited
+ulimit -s unlimited
+