From 096918c5ba8c689fdc8232b5ab45bb07b964e5e3 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Fri, 24 Jan 2025 10:23:39 +0100 Subject: [PATCH 1/4] add julia with cuda 12.6, spack 0.23 --- config.yaml | 6 ++ recipes/julia/25.1/gh200/compilers.yaml | 5 ++ recipes/julia/25.1/gh200/config.yaml | 7 ++ recipes/julia/25.1/gh200/environments.yaml | 42 +++++++++++ recipes/julia/25.1/gh200/extra/reframe.yaml | 11 +++ recipes/julia/25.1/gh200/extra/uenv_view.jl | 78 +++++++++++++++++++++ recipes/julia/25.1/gh200/modules.yaml | 23 ++++++ recipes/julia/25.1/gh200/post-install | 33 +++++++++ 8 files changed, 205 insertions(+) create mode 100644 recipes/julia/25.1/gh200/compilers.yaml create mode 100644 recipes/julia/25.1/gh200/config.yaml create mode 100644 recipes/julia/25.1/gh200/environments.yaml create mode 100644 recipes/julia/25.1/gh200/extra/reframe.yaml create mode 100644 recipes/julia/25.1/gh200/extra/uenv_view.jl create mode 100644 recipes/julia/25.1/gh200/modules.yaml create mode 100644 recipes/julia/25.1/gh200/post-install diff --git a/config.yaml b/config.yaml index 30645d1c..11abde8c 100644 --- a/config.yaml +++ b/config.yaml @@ -157,6 +157,12 @@ uenvs: zen3: "24.9/mc" gh200: "24.9/gh200" develop: False + "25.1": + recipes: + zen2: "24.9/mc" + zen3: "24.9/mc" + gh200: "24.9/gh200" + develop: False lammps: "2023": recipes: diff --git a/recipes/julia/25.1/gh200/compilers.yaml b/recipes/julia/25.1/gh200/compilers.yaml new file mode 100644 index 00000000..8209f0c2 --- /dev/null +++ b/recipes/julia/25.1/gh200/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@12.3 +gcc: + specs: + - gcc@13.2 diff --git a/recipes/julia/25.1/gh200/config.yaml b/recipes/julia/25.1/gh200/config.yaml new file mode 100644 index 00000000..482d545d --- /dev/null +++ b/recipes/julia/25.1/gh200/config.yaml @@ -0,0 +1,7 @@ +name: julia +spack: + commit: releases/v0.23 + repo: https://github.com/spack/spack.git +store: /user-environment +description: HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl, ADIOS2.jl, ...) + GNU Compiler toolchain with cray-mpich, HDF5, ADIOS2, Python, CMake and other development tools. + diff --git a/recipes/julia/25.1/gh200/environments.yaml b/recipes/julia/25.1/gh200/environments.yaml new file mode 100644 index 00000000..e88070fd --- /dev/null +++ b/recipes/julia/25.1/gh200/environments.yaml @@ -0,0 +1,42 @@ +gcc-env: + compiler: + - toolchain: gcc + spec: gcc + mpi: + spec: cray-mpich@8.1.30 + gpu: cuda + unify: true + specs: + - aws-ofi-nccl@master + - cmake + - cuda@12.6 + - fftw + - fmt + - hdf5+hl + - libtree + - meson + - nccl + - nccl-tests + - ninja + - openblas threads=openmp + - osu-micro-benchmarks@5.9 + - python + - bzip2 + - zfp + - sz + - libpng + - c-blosc2 + - libpressio + - mgard + - libcatalyst + - adios2@2.10+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst + variants: + - +mpi + - +cuda + - cuda_arch=90 + views: + default: + link: roots + uenv: + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/recipes/julia/25.1/gh200/extra/reframe.yaml b/recipes/julia/25.1/gh200/extra/reframe.yaml new file mode 100644 index 00000000..b71f2797 --- /dev/null +++ b/recipes/julia/25.1/gh200/extra/reframe.yaml @@ -0,0 +1,11 @@ +default: + features: + - cuda + - mpi + - osu-micro-benchmarks + - openmp + - serial + cc: mpicc + cxx: mpic++ + ftn: mpifort + activation: /user-environment/env/default/activate.sh diff --git a/recipes/julia/25.1/gh200/extra/uenv_view.jl b/recipes/julia/25.1/gh200/extra/uenv_view.jl new file mode 100644 index 00000000..229a0ec5 --- /dev/null +++ b/recipes/julia/25.1/gh200/extra/uenv_view.jl @@ -0,0 +1,78 @@ +# Author: Samuel Omlin, CSCS (omlins) +# +# Description: Creation of an uenv view equivalent to the activation script in JUHPC. + + +using Pkg; Pkg.add("JSON") +using JSON + + +# Check if the required environment variables are set + +function check_env(vars...) + for var in vars + if !haskey(ENV, var) + error("uenv-view: $var is not set.") + elseif isempty(ENV[var]) + error("uenv-view: $var is empty.") + end + end +end + +check_env("JULIAUP_WRAPPER_BINDIR", "JULIAUP_BINDIR", "JULIAUP_DEPOT", "JULIA_DEPOT", "JULIA_PREFDIR", "JULIAUP_INSTALLDIR", "ENV_JSON") + + +# Define the environment variables part of the julia view + +julia_view_env = Dict( + "values" => Dict( + "list" => Dict( + "PATH" => [ + Dict( + "value" => [ENV["JULIAUP_WRAPPER_BINDIR"], ENV["JULIAUP_BINDIR"]], # The wrapper must be before the juliaup bindir + "op" => "prepend", + ), + ], + ), + "scalar" => Dict( + "JULIAUP_DEPOT_PATH" => ENV["JULIAUP_DEPOT"], + "JULIA_DEPOT_PATH" => ENV["JULIA_DEPOT"], + "JULIA_LOAD_PATH" => ":$(ENV["JULIA_PREFDIR"])", # ":" means appending! + [key => ENV[env_key] for (key, env_key) in + [("CUDA_HOME", "JUHPC_CUDA_HOME"), + ("ROCM_PATH", "JUHPC_ROCM_HOME"), + ("JULIA_ADIOS2_PATH", "JUHPC_ADIOS2_HOME")] + if haskey(ENV, env_key) && !isempty(ENV[env_key])]..., # Conditional inclusion for variables if they are set and not empty + [("JULIA_CUDA_MEMORY_POOL" => "none") for env_key in ["JUHPC_CUDA_HOME"] if haskey(ENV, env_key) && !isempty(ENV[env_key])]... # Conditionally include JULIA_CUDA_MEMORY_POOL + ) + ), + "version" => 1 +) + + +# Define the julia view + +julia_view = Dict( + "julia" => Dict( + "root" => "/user-environment/env/julia", + "env" => julia_view_env, + "activate" => "/dev/null", + "description" => "description: HPC setup for juliaup, julia and some HPC key packages (juliaup and julia are installed on first execution of juliaup in $(ENV["JULIAUP_INSTALLDIR"]))", + "type" => "augment" + ) +) + + +# Merge the julia view with the existing views in the JSON file + +env = JSON.parsefile(ENV["ENV_JSON"]) +views = env["views"] +views = merge(views, julia_view) +env["views"] = views +open(ENV["ENV_JSON"],"w") do f + JSON.print(f, env, 4) +end + + +# Remove the added package +Pkg.rm("JSON") \ No newline at end of file diff --git a/recipes/julia/25.1/gh200/modules.yaml b/recipes/julia/25.1/gh200/modules.yaml new file mode 100644 index 00000000..623307b0 --- /dev/null +++ b/recipes/julia/25.1/gh200/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' diff --git a/recipes/julia/25.1/gh200/post-install b/recipes/julia/25.1/gh200/post-install new file mode 100644 index 00000000..76cc9b3d --- /dev/null +++ b/recipes/julia/25.1/gh200/post-install @@ -0,0 +1,33 @@ +#!/bin/bash + +# Author: Samuel Omlin, CSCS (omlins) +# +# Description: Definition of site specific variables and call of JUHPC. +# Site: ALPS:todi, Swiss National Supercomputing Centre (CSCS) +# Base: uenv + + +# UENV specific environment variables +export ENV_MOUNT={{ env.mount }} # export ENV_MOUNT=/user-environment +export ENV_META=$ENV_MOUNT/meta +export ENV_EXTRA=$ENV_META/extra +export ENV_JSON=$ENV_META/env.json + + +# Environment variables for HPC key packages that require system libraries (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl and ADIOS2.jl) +export JUHPC_CUDA_HOME=$(spack -C $ENV_MOUNT/config location -i cuda) +export JUHPC_CUDA_RUNTIME_VERSION=$(spack --color=never -C $ENV_MOUNT/config find cuda | \ + perl -ne 'print $1 if /cuda@([\d.]+)/') +export JUHPC_MPI_HOME=$(spack -C $ENV_MOUNT/config location -i cray-mpich) +export JUHPC_MPI_EXEC="srun -C gpu" +export JUHPC_HDF5_HOME=$(spack -C $ENV_MOUNT/config location -i hdf5) +export JUHPC_ADIOS2_HOME=$(spack -C $ENV_MOUNT/config location -i adios2) + + +# Call JUHPC +JUHPC_SETUP_INSTALLDIR=$ENV_MOUNT/juhpc_setup +JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup" +JUHPC_POST_INSTALL_JL=$ENV_EXTRA/uenv_view.jl +VERSION="v0.2.0" +wget https://raw.githubusercontent.com/JuliaParallel/JUHPC/$VERSION/juhpc -O /tmp/juhpc +bash -l /tmp/juhpc $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL From 7eb8e8aea7b238ac4665ed3dddd3a77ab5d2e4b3 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Wed, 29 Jan 2025 17:26:01 +0100 Subject: [PATCH 2/4] workaround module clash --- recipes/julia/25.1/gh200/modules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/julia/25.1/gh200/modules.yaml b/recipes/julia/25.1/gh200/modules.yaml index 623307b0..e018c58f 100644 --- a/recipes/julia/25.1/gh200/modules.yaml +++ b/recipes/julia/25.1/gh200/modules.yaml @@ -18,6 +18,6 @@ modules: autoload: none hash_length: 0 exclude_implicits: true - exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0', '%gcc@12'] projections: all: '{name}/{version}' From d51f6dd030b5af2c47ad22f9a26010f331bf6291 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Thu, 30 Jan 2025 10:02:05 +0100 Subject: [PATCH 3/4] hide implicits --- recipes/julia/25.1/gh200/modules.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/julia/25.1/gh200/modules.yaml b/recipes/julia/25.1/gh200/modules.yaml index e018c58f..76bb75be 100644 --- a/recipes/julia/25.1/gh200/modules.yaml +++ b/recipes/julia/25.1/gh200/modules.yaml @@ -17,7 +17,7 @@ modules: all: autoload: none hash_length: 0 - exclude_implicits: true - exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0', '%gcc@12'] + hide_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] projections: all: '{name}/{version}' From 7a94d22079e9f832e6c9db8a37e817706a14a00b Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 30 Jan 2025 18:09:11 +0100 Subject: [PATCH 4/4] fix build for new spack version --- config.yaml | 4 +--- recipes/julia/25.1/gh200/compilers.yaml | 2 +- recipes/julia/25.1/gh200/environments.yaml | 12 +++++++----- recipes/julia/25.1/gh200/extra/uenv_view.jl | 4 ++-- recipes/julia/25.1/gh200/modules.yaml | 3 ++- recipes/julia/25.1/gh200/post-install | 5 +++-- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/config.yaml b/config.yaml index 11abde8c..62dbf0f4 100644 --- a/config.yaml +++ b/config.yaml @@ -159,9 +159,7 @@ uenvs: develop: False "25.1": recipes: - zen2: "24.9/mc" - zen3: "24.9/mc" - gh200: "24.9/gh200" + gh200: "25.1/gh200" develop: False lammps: "2023": diff --git a/recipes/julia/25.1/gh200/compilers.yaml b/recipes/julia/25.1/gh200/compilers.yaml index 8209f0c2..25e0be1d 100644 --- a/recipes/julia/25.1/gh200/compilers.yaml +++ b/recipes/julia/25.1/gh200/compilers.yaml @@ -2,4 +2,4 @@ bootstrap: spec: gcc@12.3 gcc: specs: - - gcc@13.2 + - gcc@13.3 diff --git a/recipes/julia/25.1/gh200/environments.yaml b/recipes/julia/25.1/gh200/environments.yaml index e88070fd..c54a2bb9 100644 --- a/recipes/julia/25.1/gh200/environments.yaml +++ b/recipes/julia/25.1/gh200/environments.yaml @@ -1,14 +1,13 @@ gcc-env: compiler: - toolchain: gcc - spec: gcc + spec: gcc@13 mpi: spec: cray-mpich@8.1.30 gpu: cuda unify: true specs: - - aws-ofi-nccl@master - - cmake + - cmake@3.27 - cuda@12.6 - fftw - fmt @@ -26,10 +25,13 @@ gcc-env: - sz - libpng - c-blosc2 - - libpressio + - libpressio@0.95.1 - mgard - libcatalyst - - adios2@2.10+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst + - adios2@2.10+hdf5+python+fortran + - 'adios2@2.10+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+mgard+libcatalyst' + # disable libpressio because it causes compilation errors when building adios2 + #- 'adios2@2.10+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst' variants: - +mpi - +cuda diff --git a/recipes/julia/25.1/gh200/extra/uenv_view.jl b/recipes/julia/25.1/gh200/extra/uenv_view.jl index 229a0ec5..b855a83c 100644 --- a/recipes/julia/25.1/gh200/extra/uenv_view.jl +++ b/recipes/julia/25.1/gh200/extra/uenv_view.jl @@ -53,7 +53,7 @@ julia_view_env = Dict( # Define the julia view julia_view = Dict( - "julia" => Dict( + "juliaup" => Dict( "root" => "/user-environment/env/julia", "env" => julia_view_env, "activate" => "/dev/null", @@ -75,4 +75,4 @@ end # Remove the added package -Pkg.rm("JSON") \ No newline at end of file +Pkg.rm("JSON") diff --git a/recipes/julia/25.1/gh200/modules.yaml b/recipes/julia/25.1/gh200/modules.yaml index 76bb75be..d4510ea6 100644 --- a/recipes/julia/25.1/gh200/modules.yaml +++ b/recipes/julia/25.1/gh200/modules.yaml @@ -15,8 +15,9 @@ modules: tcl: /user-environment/modules tcl: all: - autoload: none + autoload: run hash_length: 0 + exclude_implicits: true hide_implicits: true exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] projections: diff --git a/recipes/julia/25.1/gh200/post-install b/recipes/julia/25.1/gh200/post-install index 76cc9b3d..e85014b1 100644 --- a/recipes/julia/25.1/gh200/post-install +++ b/recipes/julia/25.1/gh200/post-install @@ -19,14 +19,15 @@ export JUHPC_CUDA_HOME=$(spack -C $ENV_MOUNT/config location -i cuda) export JUHPC_CUDA_RUNTIME_VERSION=$(spack --color=never -C $ENV_MOUNT/config find cuda | \ perl -ne 'print $1 if /cuda@([\d.]+)/') export JUHPC_MPI_HOME=$(spack -C $ENV_MOUNT/config location -i cray-mpich) -export JUHPC_MPI_EXEC="srun -C gpu" +export JUHPC_MPI_EXEC="srun" export JUHPC_HDF5_HOME=$(spack -C $ENV_MOUNT/config location -i hdf5) export JUHPC_ADIOS2_HOME=$(spack -C $ENV_MOUNT/config location -i adios2) # Call JUHPC JUHPC_SETUP_INSTALLDIR=$ENV_MOUNT/juhpc_setup -JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup" +#JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup" +JULIAUP_INSTALLDIR="\$SCRATCH/gh200/juliaup" JUHPC_POST_INSTALL_JL=$ENV_EXTRA/uenv_view.jl VERSION="v0.2.0" wget https://raw.githubusercontent.com/JuliaParallel/JUHPC/$VERSION/juhpc -O /tmp/juhpc