From d9f0026c9884a0685296ec06aaadf1ce8f6541b2 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli <1237199+simonpintarelli@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:57:20 +0100 Subject: [PATCH] split cray mpich packages in mpich, pmi, pals, gtl (#156) = split cray-mpich into cray-mpich, cray-gtl, cray-pals, cray-pmi - use a single archive for cray-mpich, copy either gcc or nvhpc folder during installation - add all mpich versions since 8.1.18 --- stackinator/repo/packages/cray-gtl/package.py | 111 +++++++++++ .../repo/packages/cray-mpich/package.py | 186 +++++------------- .../repo/packages/cray-pals/package.py | 78 ++++++++ stackinator/repo/packages/cray-pmi/package.py | 94 +++++++++ 4 files changed, 330 insertions(+), 139 deletions(-) create mode 100644 stackinator/repo/packages/cray-gtl/package.py create mode 100644 stackinator/repo/packages/cray-pals/package.py create mode 100644 stackinator/repo/packages/cray-pmi/package.py diff --git a/stackinator/repo/packages/cray-gtl/package.py b/stackinator/repo/packages/cray-gtl/package.py new file mode 100644 index 0000000..2fbe8c9 --- /dev/null +++ b/stackinator/repo/packages/cray-gtl/package.py @@ -0,0 +1,111 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import spack.compilers +from spack.package import * + + +class CrayGtl(Package): + """Install cray-gtl""" + + homepage = "https://www.hpe.com/us/en/compute/hpc/hpc-software.html" + url = "https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-gtl-8.1.26.tar.gz" + maintainers = ["bcumming"] + + version( + "8.1.28", + sha256="2e82c618648e79bdc4b8bf9394be8fd59c34ccd77f172afd11fce38beca1ecab", + ) + version( + "8.1.27", + sha256="c00836641c7d1f40300f8e4b068a1ba1ff4b0ca31af24b9507d768fc6fe5753c", + ) + version( + "8.1.26", + sha256="37d9626cb5f851f63c9799c18a419354c6f21c77f90558472552156df9eef311", + ) + version( + "8.1.25", + sha256="a2e2af2037e63b64ef74d870c0bab91a8109e75eef82a30250b81b0d785ff6ae", + ) + version( + "8.1.24", + sha256="2fa8635f829e67844e7b30dffb092a336d257e0e769d2225030f2ccf4c1d302f", + ) + version( + "8.1.23", + sha256="034667c2ea49eec76ef8f79494231bad94884b99683edabf781beed01ec681e4", + ) + version( + "8.1.21", + sha256="78072edfcb6cc24cfefab06e824111b5b2b839551235ece68cd154bec7936a24", + ) + version( + "8.1.18", + sha256="5ac6b0877fd0f6afaaf391fffef41daf4d3150edc3250721c9abd4ded6b58486", + ) + + variant("cuda", default=False) + variant("rocm", default=False) + conflicts("+cuda", when="+rocm", msg="Pick either CUDA or ROCM") + + # Fix up binaries with patchelf. + depends_on("patchelf", type="build") + + conflicts("+cuda", when="+rocm", msg="Pick either CUDA or ROCM") + + with when("+cuda"): + depends_on("cuda@11.0:11", type="link", when="@:8.1.27") + depends_on("cuda@12:", type="link", when="@8.1.28:") + + with when("+rocm"): + # libamdhip64.so.5 + depends_on("hip@5:", type="link") + # libhsa-runtime64.so.1 + depends_on("hsa-rocr-dev", type="link") + + def get_rpaths(self): + # Those rpaths are already set in the build environment, so + # let's just retrieve them. + pkgs = os.getenv("SPACK_RPATH_DIRS", "").split(":") + compilers = os.getenv("SPACK_COMPILER_IMPLICIT_RPATHS", "").split(":") + return ":".join([p for p in pkgs + compilers if p]) + + def should_patch(self, file): + # Returns true if non-symlink ELF file. + if os.path.islink(file): + return False + try: + with open(file, "rb") as f: + return f.read(4) == b"\x7fELF" + except OSError: + return False + + def install(self, spec, prefix): + install_tree(".", prefix) + + @property + def libs(self): + if "+cuda" in self.spec: + return find_libraries("libmpi_gtl_cuda", root=self.prefix, shared=True) + if "+rocm" in self.spec: + return find_libraries("libmpi_gtl_hsa", root=self.prefix, shared=True) + + @run_after("install") + def fixup_binaries(self): + patchelf = which("patchelf") + rpath = self.get_rpaths() + for root, _, files in os.walk(self.prefix): + for name in files: + f = os.path.join(root, name) + if not self.should_patch(f): + continue + patchelf("--force-rpath", "--set-rpath", rpath, f, fail_on_error=False) + # The C compiler wrapper can fail because libmpi_gtl_cuda refers to the symbol + # __gxx_personality_v0 but wasn't linked against libstdc++. + if "libmpi_gtl_cuda.so" in str(f): + patchelf("--add-needed", "libstdc++.so", f, fail_on_error=False) diff --git a/stackinator/repo/packages/cray-mpich/package.py b/stackinator/repo/packages/cray-mpich/package.py index 3adad7c..41eff56 100644 --- a/stackinator/repo/packages/cray-mpich/package.py +++ b/stackinator/repo/packages/cray-mpich/package.py @@ -15,58 +15,40 @@ class CrayMpich(Package): """Intended to override the main cray-mpich""" homepage = "https://www.hpe.com/us/en/compute/hpc/hpc-software.html" - url = "https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.18.4-gcc.tar.gz" - maintainers = ["haampie"] + url = "https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.26.tar.gz" + maintainers = ["bcumming"] version( - "8.1.25-gcc", - sha256="95a8a161dc9704ea7b971dc8c1b7ec4d63de57e2f6932f0aa3d1ff1d73899765", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.25-gcc.tar.gz", + "8.1.28", + sha256="935fca183dabfcae1a4cfc664234537f25f1e87fb2765f32636be7352514f476", ) version( - "8.1.25-nvhpc", - sha256="7a89a3f5d35538a4f7984c1403ca888e1b018485597318eaefa4639341e1eb27", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.25-nvhpc.tar.gz", + "8.1.27", + sha256="c555f180cbe7272acd76e3e55cd9150c2b20b8ec263228f64fe2b682eec1c612", ) version( - "8.1.24-gcc", - sha256="3da0e421c3faaadbe18e57dd033b0ec6513e0d9ed7fbfa77f05a02bada4cd483", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.24-gcc.tar.gz", + "8.1.26", + sha256="3c23cfe24b8f05e0c68a059919ac7dd77c45a333cad9aea41872a51d256b12d1", ) version( - "8.1.24-nvhpc", - sha256="1b507f4e9150cf188a0571aad0d190fc8ee981def1d6198c998673d73828ed6f", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.24-nvhpc.tar.gz", + "8.1.25", + sha256="46e8c2804f5d34815bcf381e1957e749e6d7b286853bd94ae2bff73a6df39263", ) version( - "8.1.23-gcc", - sha256="2d1dfda811848d278548b0d7735f17341c70380dbf7f91dc680e5afcfb5e0038", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.23-gcc.tar.gz", + "8.1.24", + sha256="96876331bb7098e9ef2eea2c8bb25e479838c48b294ae5790094c19644e73a7d", ) version( - "8.1.23-nvhpc", - sha256="1dd9b161c538dbac564ecff6f1552220ba40dcc9436dc855087438f29861eba1", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.23-nvhpc.tar.gz", + "8.1.23", + sha256="c0985424ef376b29e6f1b9c2016b8cfe6430b9ce434da9700a6c14433b47cf20", ) version( - "8.1.21.1-gcc", - sha256="0a6852ebf06afd249285fd09566e8489300cba96ad66e90c40df36b6af9a631e", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.21.1-gcc.tar.gz", + "8.1.21", + sha256="8b4e0ff9cba48ef7dcd4dd8092b35bb6456420de2dcf7b4f05d7c69f8e266de3", ) version( - "8.1.21.1-nvhpc", - sha256="791b39f2ecb933060abaa8c8704e71da01c6962c4211cc99d12b9d964e9be4cb", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.21.1-nvhpc.tar.gz", - ) - version( - "8.1.18.4-gcc", - sha256="776c695aeed62b3f64a1bca11b30a2537a907777a8664d2f092e3deac288e4ad", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.18.4-gcc.tar.gz", - ) - version( - "8.1.18.4-nvhpc", - sha256="2285433363c75a04ccdf4798be5b0e296e0c9a8fb8fcb38eb0aa4ccf8d1e0843", - url="https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-mpich-8.1.18.4-nvhpc.tar.gz", + "8.1.18", + sha256="45d519be217cea89c58893d25826b15d99183247d15ecee9c3f64913660d79c2", ) variant("cuda", default=False) @@ -78,95 +60,28 @@ class CrayMpich(Package): # Fix up binaries with patchelf. depends_on("patchelf", type="build") - with when("+cuda"): - # libcudart.so.11.0 - depends_on("cuda@11.0:11", type="link") - with when("+rocm"): - # libamdhip64.so.5 - depends_on("hip@5:", type="link") - # libhsa-runtime64.so.1 - depends_on("hsa-rocr-dev", type="link") + for ver in [ + "8.1.18", + "8.1.21", + "8.1.23", + "8.1.24", + "8.1.25", + "8.1.26", + "8.1.27", + "8.1.28", + ]: + with when("+cuda"): + depends_on(f"cray-gtl@{ver} +cuda", type="link", when="@" + ver) + with when("+rocm"): + depends_on(f"cray-gtl@{ver} +rocm", type="link", when="@" + ver) - # libfabric.so.1 depends_on("libfabric@1:", type="link") - with when("@8.1.25-gcc"): - # libgfortran.so.5 - conflicts("%gcc@:7") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "gcc": - conflicts("%{}".format(__compiler), msg="gcc required") - - with when("@8.1.25-nvhpc"): - conflicts("%nvhpc@:20.7") - conflicts("+rocm") - conflicts("~cuda") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "nvhpc": - conflicts("%{}".format(__compiler), msg="nvhpc required") - - with when("@8.1.24-gcc"): - # libgfortran.so.5 - conflicts("%gcc@:7") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "gcc": - conflicts("%{}".format(__compiler), msg="gcc required") - - with when("@8.1.24-nvhpc"): - conflicts("%nvhpc@:20.7") - conflicts("+rocm") - conflicts("~cuda") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "nvhpc": - conflicts("%{}".format(__compiler), msg="nvhpc required") - - with when("@8.1.23-gcc"): - # libgfortran.so.5 - conflicts("%gcc@:7") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "gcc": - conflicts("%{}".format(__compiler), msg="gcc required") - - with when("@8.1.23-nvhpc"): - conflicts("%nvhpc@:20.6") - conflicts("+rocm") - conflicts("~cuda") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "nvhpc": - conflicts("%{}".format(__compiler), msg="nvhpc required") - - with when("@8.1.21.1-gcc"): - # libgfortran.so.5 - conflicts("%gcc@:7") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "gcc": - conflicts("%{}".format(__compiler), msg="gcc required") - - with when("@8.1.21.1-nvhpc"): - conflicts("%nvhpc@:20.6") - conflicts("+rocm") - conflicts("~cuda") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "nvhpc": - conflicts("%{}".format(__compiler), msg="nvhpc required") - - with when("@8.1.18.4-gcc"): - # libgfortran.so.5 - conflicts("%gcc@:7") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "gcc": - conflicts("%{}".format(__compiler), msg="gcc required") - - with when("@8.1.18.4-nvhpc"): - conflicts("%nvhpc@:20.6") - conflicts("+rocm") - conflicts("~cuda") - for __compiler in spack.compilers.supported_compilers(): - if __compiler != "nvhpc": - conflicts("%{}".format(__compiler), msg="nvhpc required") - - # TODO: libpals.so.0? no clue where it comes from. + depends_on("cray-pmi", type="link") + + conflicts("%gcc@:7") + conflicts("%gcc@:11", when="@8.1.28:") def setup_run_environment(self, env): env.set("MPICC", join_path(self.prefix.bin, "mpicc")) @@ -204,7 +119,10 @@ def should_patch(self, file): return False def install(self, spec, prefix): - install_tree(".", prefix) + if "%nvhpc" in self.spec: + install_tree("mpich-nvhpc", prefix) + elif "%gcc" in self.spec: + install_tree("mpich-gcc", prefix) @run_after("install") def fixup_binaries(self): @@ -216,10 +134,6 @@ def fixup_binaries(self): if not self.should_patch(f): continue patchelf("--force-rpath", "--set-rpath", rpath, f, fail_on_error=False) - # The C compiler wrapper can fail because libmpi_gtl_cuda refers to the symbol - # __gxx_personality_v0 but wasn't linked against libstdc++. - if "libmpi_gtl_cuda.so" in str(f): - patchelf("--add-needed", "libstdc++.so", f, fail_on_error=False) @run_after("install") def fixup_compiler_paths(self): @@ -233,12 +147,14 @@ def fixup_compiler_paths(self): # link with the relevant gtl lib if "+cuda" in self.spec: - gtl_library = "-lmpi_gtl_cuda" + lpath = self.spec["cray-gtl"].prefix.lib + gtl_library = f"-L{lpath} -lmpi_gtl_cuda" elif "+rocm" in self.spec: - gtl_library = "-lmpi_gtl_hsa" + lpath = self.spec["cray-gtl"].prefix.lib + gtl_library = f"-L{lpath} -lmpi_gtl_hsa" else: gtl_library = "" - + print("==== GTL_LIBRARY", gtl_library) filter_file("@@GTL_LIBRARY@@", gtl_library, self.prefix.bin.mpicc, string=True) filter_file("@@GTL_LIBRARY@@", gtl_library, self.prefix.bin.mpicxx, string=True) filter_file( @@ -252,9 +168,7 @@ def headers(self): "cray_version", self.prefix.include, recursive=True ) # cray_version.h # cray-mpich depends on cray-pmi - hdrs += find_headers( - "pmi", self.prefix.include, recursive=True - ) # See cray-pmi package + # hdrs += find_headers("pmi", self.prefix.include, recursive=True) # See cray-pmi package hdrs.directories = os.path.dirname(hdrs[0]) return hdrs @@ -270,17 +184,11 @@ def libs(self): if "f90" in query_parameters: libraries.extend(["libmpif90", "libmpichf90"]) - if "+cuda" in self.spec: - libraries.append("libmpi*cuda") - - if "+rocm" in self.spec: - libraries.append("libmpi*hsa") - libs = [] for lib_folder in [self.prefix.lib, self.prefix.lib64]: libs += find_libraries(libraries, root=lib_folder, recursive=True) # cray-mpich depends on cray-pmi - libs += find_libraries("libpmi", root=lib_folder, recursive=True) + # libs += find_libraries("libpmi", root=lib_folder, recursive=True) libs += find_libraries("libopa", root=lib_folder, recursive=True) libs += find_libraries("libmpl", root=lib_folder, recursive=True) diff --git a/stackinator/repo/packages/cray-pals/package.py b/stackinator/repo/packages/cray-pals/package.py new file mode 100644 index 0000000..5fa7372 --- /dev/null +++ b/stackinator/repo/packages/cray-pals/package.py @@ -0,0 +1,78 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import spack.compilers +from spack.package import * + + +class CrayPals(Package): + """Install cray-pals""" + + """Intended to override the main cray-pals""" + + homepage = "https://www.hpe.com/us/en/compute/hpc/hpc-software.html" + url = "https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-pals-1.2.12.tar.gz" + maintainers = ["simonpintarelli"] + + version( + "1.3.2", + sha256="deea749476de0f545b31fcd0912f133d7ba60b84f673e47d8b4b15d5a117254c", + ) + version( + "1.2.12", + sha256="c94d29c09ed650c4e98a236df7ced77f027bdf987919a91a1a1382f704a85bb9", + ) + version( + "1.2.11", + sha256="e1af09e39d70e28381de806548c6cb29c23abf891a078f46eb71c301a3f0994c", + ) + version( + "1.2.9", + sha256="ceec6f99bea9df3f7f657a7df499445e62976064dda3f3e437d61e895ec31601", + ) + version( + "1.2.5", + sha256="d7269ed8f4deab816e3d4006090ec68b25ccc585200d16728ed9a914baf4d9bf", + ) + version( + "1.2.4", + sha256="a253939585bad2bb9061b98be6e517f18bda0602ecfd38f75c734a01d12003f2", + ) + + # Fix up binaries with patchelf. + depends_on("patchelf", type="build") + + def get_rpaths(self): + # Those rpaths are already set in the build environment, so + # let's just retrieve them. + pkgs = os.getenv("SPACK_RPATH_DIRS", "").split(":") + compilers = os.getenv("SPACK_COMPILER_IMPLICIT_RPATHS", "").split(":") + return ":".join([p for p in pkgs + compilers if p]) + + def should_patch(self, file): + # Returns true if non-symlink ELF file. + if os.path.islink(file): + return False + try: + with open(file, "rb") as f: + return f.read(4) == b"\x7fELF" + except OSError: + return False + + def install(self, spec, prefix): + install_tree(".", prefix) + + @run_after("install") + def fixup_binaries(self): + patchelf = which("patchelf") + rpath = self.get_rpaths() + for root, _, files in os.walk(self.prefix): + for name in files: + f = os.path.join(root, name) + if not self.should_patch(f): + continue + patchelf("--force-rpath", "--set-rpath", rpath, f, fail_on_error=False) diff --git a/stackinator/repo/packages/cray-pmi/package.py b/stackinator/repo/packages/cray-pmi/package.py new file mode 100644 index 0000000..633cdfc --- /dev/null +++ b/stackinator/repo/packages/cray-pmi/package.py @@ -0,0 +1,94 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import spack.compilers +from spack.package import * + + +class CrayPmi(Package): + """Install cray-pmi""" + + """Intended to override the main cray-pmi""" + + homepage = "https://www.hpe.com/us/en/compute/hpc/hpc-software.html" + url = "https://jfrog.svc.cscs.ch/artifactory/cray-mpich/cray-pmi-6.1.11.tar.gz" + maintainers = ["bcumming"] + + version( + "6.1.13", + sha256="217ac554cf84a4c7f08cd149c6a18428e1e3533d73e350fa291b6800895b632e", + ) + version( + "6.1.12", + sha256="0f1caa93c881e1a5a4b5a65d1cb3a04d9c549ffdb9524b53a6e7ca9317dd90ee", + ) + version( + "6.1.11", + sha256="5ebcece6a610da02cd41a9a386fd7463ee909bd55e3370d6d372603f90be9afe", + ) + version( + "6.1.10", + sha256="f4fbe75c201a171dcfe6ada773a4bf0c606767a0b7a8a76fd19d10852abe1290", + ) + version( + "6.1.9", + sha256="8fd4194c6c5167f8b81b1cf9b76341669e40d647d0caecef287be6f0f5d95290", + ) + version( + "6.1.8", + sha256="6c7e5d3038e26b9d0e82428b25b570d00401a6fc9f2fd3c008f15a253a8e2305", + ) + version( + "6.1.7", + sha256="574b21bd6f8970521c2bc4f096aced896fec8b749f854272cc7bbb7130ae92d8", + ) + + # Fix up binaries with patchelf. + depends_on("patchelf", type="build") + + depends_on("cray-pals@1.3.2", type="link", when="@6.1.13:") + depends_on("cray-pals@1.2.12", type="link", when="@6.1.11:6.1.12") + depends_on("cray-pals@1.2.11", type="link", when="@6.1.10") + depends_on("cray-pals@1.2.9", type="link", when="@6.1.9") + depends_on("cray-pals@1.2.5", type="link", when="@6.1.8") + depends_on("cray-pals@1.2.4", type="link", when="@6.1.7") + depends_on("cray-pals@1.2.0", type="link", when="@6.0.17") + + def get_rpaths(self): + # Those rpaths are already set in the build environment, so + # let's just retrieve them. + pkgs = os.getenv("SPACK_RPATH_DIRS", "").split(":") + compilers = os.getenv("SPACK_COMPILER_IMPLICIT_RPATHS", "").split(":") + return ":".join([p for p in pkgs + compilers if p]) + + def should_patch(self, file): + # Returns true if non-symlink ELF file. + if os.path.islink(file): + return False + try: + with open(file, "rb") as f: + return f.read(4) == b"\x7fELF" + except OSError: + return False + + def install(self, spec, prefix): + install_tree(".", prefix) + + @property + def libs(self): + return find_libraries(["libmpi", "libpmi2"], root=self.prefix, shared=True) + + @run_after("install") + def fixup_binaries(self): + patchelf = which("patchelf") + rpath = self.get_rpaths() + for root, _, files in os.walk(self.prefix): + for name in files: + f = os.path.join(root, name) + if not self.should_patch(f): + continue + patchelf("--force-rpath", "--set-rpath", rpath, f, fail_on_error=False)