Merge pull request #9 from boeschf/add_ghex2

Update spack recipes

.github/workflows/common.yml

@@ -0,0 +1,115 @@
+name: common
+
+on:
+  workflow_call:
+    inputs:
+      package-name:
+        required: true
+        type: string
+      package-version:
+        required: true
+        type: string
+      package-variants:
+        required: false
+        type: string
+        default: ''
+      test-package:
+        required: false
+        type: boolean
+        default: true
+
+jobs:
+  setup-and-test:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      matrix:
+        spack-version: ['v0.23.0', 'develop-2025-01-19']
+
+    env:
+      REPO_PATH: ghex-org-spack-packages
+      SPACK_PATH: spack
+
+      fail-fast: false
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "OS_NAME=ubuntu" >> $GITHUB_ENV
+          echo "OS_VERSION=$(lsb_release -rs)" >> $GITHUB_ENV
+          echo "OMPI_ALLOW_RUN_AS_ROOT=1" >>  $GITHUB_ENV
+          echo "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >>  $GITHUB_ENV
+
+      - name: Install additional Ubuntu packages
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y gfortran libblas-dev
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ env.REPO_PATH }}
+
+      - name: Configure ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.spack-version }}
+
+      - name: Clone Spack and setup environment
+        run: |
+          git clone -c feature.manyFiles=true --depth 1 --branch ${{ matrix.spack-version }} https://github.com/spack/spack.git ${{ env.SPACK_PATH }}
+          . ${{ env.SPACK_PATH }}/share/spack/setup-env.sh
+          spack config --scope site add config:ccache:true
+          spack config --scope site add config:install_tree:padded_length:128
+          spack config --scope site add packages:all:require:'target=x86_64_v3'
+          cat <<EOF > ${{ env.SPACK_PATH }}/etc/spack/mirrors.yaml
+          mirrors:
+            ghcr-buildcache:
+              url: oci://ghcr.io/ghex-org/spack-${{ matrix.spack-version }}-buildcache-${{ env.OS_NAME }}-${{ env.OS_VERSION }}
+              signed: false
+              access_pair:
+                id_variable: GITHUB_USER
+                secret_variable: GITHUB_TOKEN
+          EOF
+
+          spack compiler find
+          spack repo add ${{ env.REPO_PATH }}
+          spack reindex
+          spack buildcache list
+
+      - name: Build dependencies
+        run: |
+          . ${{ env.SPACK_PATH }}/share/spack/setup-env.sh
+          echo "spack spec -I ${{ inputs.package-name }}@${{ inputs.package-version }} ${{ inputs.package-variants }}"
+          spack spec -I ${{ inputs.package-name }}@${{ inputs.package-version }} ${{ inputs.package-variants }}
+          spack install --only dependencies ${{ inputs.package-name }}@${{ inputs.package-version }} ${{ inputs.package-variants }}
+
+      - name: Push packages to buildcache
+        env:
+          GITHUB_USER: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          . ${{ env.SPACK_PATH }}/share/spack/setup-env.sh
+          spack --color never find --format '{/hash};' | sed 's/;[[:space:]]*//g' | xargs -I{} spack buildcache push --base-image ${{ env.OS_NAME }}:${{ env.OS_VERSION }} --only package ghcr-buildcache  {}
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+
+      - name: Build package without testing
+        if: ${{ !inputs.test-package }}
+        run: |
+          . ${{ env.SPACK_PATH }}/share/spack/setup-env.sh
+          spack install --verbose --show-log-on-error ${{ inputs.package-name }}@${{ inputs.package-version }} ${{ inputs.package-variants }}
+
+      - name: Build and test package
+        if: ${{ inputs.test-package }}
+        run: |
+          . ${{ env.SPACK_PATH }}/share/spack/setup-env.sh
+          spack install --test=root --verbose --show-log-on-error ${{ inputs.package-name }}@${{ inputs.package-version }} ${{ inputs.package-variants }}

.github/workflows/ghex.yml

@@ -0,0 +1,34 @@
+name: ghex
+
+on:
+  push:
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: '0 8 * * 1'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  test-ghex:
+    uses: ./.github/workflows/common.yml
+    strategy:
+      matrix:
+        package-version: ['0.4.1', 'master']
+        compiler: ['[email protected]']
+        backend: ['mpi', 'ucx']
+        python-bindings: [true]
+        device: ['cpu']
+    with:
+      package-name: ghex
+      package-version: ${{ matrix.package-version }}
+      package-variants: >-
+        %${{ matrix.compiler }}
+        backend=${{ matrix.backend }}
+        ${{ matrix.python-bindings && '+python' || '' }}
+        ${{ matrix.device == 'cuda' && '+cuda' || '' }}
+        ${{ matrix.device == 'rocm' && '+rocm' || '' }}
+      test-package: ${{ matrix.device == 'cpu' }}

.github/workflows/hwmalloc.yml

@@ -1,24 +1,30 @@
-name: test-hwmalloc
+name: hwmalloc
 
 on:
   push:
   pull_request:
     branches:
       - main
+  schedule:
+    - cron: '0 8 * * 1'
+
+permissions:
+  contents: read
+  packages: write
 
 jobs:
-  spack-install:
-    runs-on: ubuntu-latest
-    container: boeschf/ghex-org
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-      - name: build and test
-        run: |
-          . /opt/spack/share/spack/setup-env-clean.sh
-          spack repo add .
-          spack spec -I hwmalloc
-          spack install --test=root --verbose --show-log-on-error hwmalloc
+  test-hwmalloc:
+    uses: ./.github/workflows/common.yml
+    strategy:
+      matrix:
+        package-version: ['0.3.0', 'master']
+        compiler: ['[email protected]']
+        device: ['cpu', 'cuda']
+    with:
+      package-name: hwmalloc
+      package-version: ${{ matrix.package-version }}
+      package-variants: >-
+        %${{ matrix.compiler }}
+        ${{ matrix.device == 'cuda' && '+cuda cuda_arch=90' || '' }}
+        ${{ matrix.device == 'rocm' && '+rocm amdgpu_target=gfx90a' || '' }}
+      test-package: ${{ matrix.device == 'cpu' }}

.github/workflows/oomph.yml

@@ -1,30 +1,34 @@
-name: test-oomph
+name: oomph
 
 on:
   push:
   pull_request:
     branches:
       - main
+  schedule:
+    - cron: '0 8 * * 1'
+
+permissions:
+  contents: read
+  packages: write
 
 jobs:
-  spack-install:
-    runs-on: ubuntu-latest
-    container: boeschf/ghex-org
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-      # export OpenMPI environment variables: allow to run as root
-      - name: set env
-        run: |
-          echo "OMPI_ALLOW_RUN_AS_ROOT=1" >>  $GITHUB_ENV
-          echo "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >>  $GITHUB_ENV
-      - name: build and test
-        run: |
-          . /opt/spack/share/spack/setup-env-clean.sh
-          spack repo add .
-          spack spec -I oomph
-          # need `--dirty` here for environment variables to propagate
-          spack install --dirty --test=root --verbose --show-log-on-error oomph
+  test-oomph:
+    uses: ./.github/workflows/common.yml
+    strategy:
+      matrix:
+        package-version: ['0.4.0', 'main']
+        compiler: ['[email protected]']
+        backend: ['mpi', 'ucx']
+        fortran-bindings: [false]
+        device: ['cpu', 'cuda']
+    with:
+      package-name: oomph
+      package-version: ${{ matrix.package-version }}
+      package-variants: >-
+        %${{ matrix.compiler }}
+        backend=${{ matrix.backend }}
+        ${{ matrix.fortran-bindings && '+fortran-bindings' || '' }}
+        ${{ matrix.device == 'cuda' && '+cuda cuda_arch=90' || '' }}
+        ${{ matrix.device == 'rocm' && '+rocm amdgpu_target=gfx942' || '' }}
+      test-package: ${{ matrix.device == 'cpu' }}

README.md

@@ -1,5 +1,6 @@
 [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 [![Test: hwmalloc](https://github.com/ghex-org/spack-repos/actions/workflows/hwmalloc.yml/badge.svg)](https://github.com/ghex-org/spack-repos/actions/workflows/hwmalloc.yml)
 [![Test: oomph](https://github.com/ghex-org/spack-repos/actions/workflows/oomph.yml/badge.svg)](https://github.com/ghex-org/spack-repos/actions/workflows/oomph.yml)
+[![Test: ghex](https://github.com/ghex-org/spack-repos/actions/workflows/ghex.yml/badge.svg)](https://github.com/ghex-org/spack-repos/actions/workflows/ghex.yml)
 # spack-repos
 Spack packages for ghex-org repositories

packages/ghex/package.py

@@ -0,0 +1,87 @@
+from spack.package import *
+
+
+class Ghex(CMakePackage, CudaPackage, ROCmPackage):
+    """GHEX is a generic halo-exchange library."""
+
+    homepage = "https://github.com/ghex-org/GHEX"
+    url = "https://github.com/ghex-org/GHEX/archive/refs/tags/v0.3.0.tar.gz"
+    git = "https://github.com/ghex-org/GHEX.git"
+    maintainers = ["boeschf"]
+
+    version("0.4.1", tag="v0.4.1", submodules=True)
+    version("0.4.0", tag="v0.4.0", submodules=True)
+    version("0.3.0", tag="v0.3.0", submodules=True)
+    version("master", branch="master", submodules=True)
+
+    depends_on("cxx", type="build")
+
+    generator("ninja")
+
+    backends = ("mpi", "ucx", "libfabric")
+    variant(
+        "backend", default="mpi", description="Transport backend", values=backends, multi=False
+    )
+    variant("xpmem", default=False, description="Use xpmem shared memory")
+    variant("python", default=True, description="Build Python bindings")
+
+    depends_on("[email protected]:", type="build")
+    depends_on("mpi")
+    depends_on("boost")
+    depends_on("xpmem", when="+xpmem", type=("build", "run"))
+
+    depends_on("oomph")
+    for backend in backends:
+        depends_on(f"oomph backend={backend}", when=f"backend={backend}")
+    depends_on("oomph+cuda", when="+cuda")
+    depends_on("oomph+rocm", when="+rocm")
+    depends_on("[email protected]:", when="@0.3:")
+
+    conflicts("+cuda+rocm")
+
+    with when("+python"):
+        extends("python")
+        depends_on("[email protected]:", type="build")
+        depends_on("py-pip", type="build")
+        depends_on("py-pybind11", type="build")
+        depends_on("py-mpi4py", type=("build", "run"))
+        depends_on("py-numpy", type=("build", "run"))
+
+        depends_on("py-pytest", when="+python", type=("test"))
+
+    def cmake_args(self):
+        spec = self.spec
+
+        args = [
+            self.define("GHEX_USE_BUNDLED_LIBS", True),
+            self.define("GHEX_USE_BUNDLED_GRIDTOOLS", True),
+            self.define("GHEX_USE_BUNDLED_GTEST", self.run_tests),
+            self.define("GHEX_USE_BUNDLED_OOMPH", False),
+            self.define("GHEX_TRANSPORT_BACKEND", spec.variants["backend"].value.upper()),
+            self.define_from_variant("GHEX_USE_XPMEM", "xpmem"),
+            self.define_from_variant("GHEX_BUILD_PYTHON_BINDINGS", "python"),
+            self.define("GHEX_WITH_TESTING", self.run_tests),
+        ]
+
+        if spec.satisfies("+python"):
+            args.append(self.define("GHEX_PYTHON_LIB_PATH", python_platlib))
+
+        if self.run_tests and spec.satisfies("^openmpi"):
+            args.append(self.define("MPIEXEC_PREFLAGS", "--oversubscribe"))
+
+        if "+cuda" in spec and spec.variants["cuda_arch"].value != "none":
+            arch_str = ";".join(spec.variants["cuda_arch"].value)
+            args.append(self.define("CMAKE_CUDA_ARCHITECTURES", arch_str))
+            args.append(self.define("GHEX_USE_GPU", True))
+            args.append(self.define("GHEX_GPU_TYPE", "CUDA"))
+
+        if "+rocm" in spec and spec.variants["amdgpu_target"].value != "none":
+            arch_str = ";".join(spec.variants["amdgpu_target"].value)
+            args.append(self.define("CMAKE_HIP_ARCHITECTURES", arch_str))
+            args.append(self.define("GHEX_USE_GPU", True))
+            args.append(self.define("GHEX_GPU_TYPE", "AMD"))
+
+        if spec.satisfies("~cuda~rocm"):
+            args.append(self.define("GHEX_USE_GPU", False))
+
+        return args