diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 65e76df82..bfdd8dc7e 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -35,6 +35,9 @@ standard or later. The following MPI implementations should work out-of-the-box - [Fujitsu MPI](https://www.fujitsu.com/global/about/resources/publications/technicalreview/2020-03/article07.html#cap-03) - [HPE MPT/HMPT](https://support.hpe.com/hpesc/public/docDisplay?docLocale=en_US&docId=a00105727en_us) +If you are using an MPI implementation that is not ABI-compatible with any one +of these, please read the section on [Supporting an unknown ABI](@ref) below. + ### Configuration Run `MPIPreferences.use_system_binary()`. This will attempt to locate and to identify any available MPI implementation, and create a file called `LocalPreferences.toml` adjacent to the current `Project.toml`. @@ -100,6 +103,50 @@ Preferences are merged across the Julia load path, such that it is feasible to p that will take precedent by modifying the local `Project.toml` or by providing a `LocalPreferences.toml` file. + +### Supporting an unknown ABI +If you want to use an MPI implementation not officially supported by MPI.jl, you +need to create your own ABI file with all relevant MPI constants. The files for supported +ABIs are stored in the `src/consts/` folder, e.g., +[`mpich.jl`](https://github.com/JuliaParallel/MPI.jl/blob/master/src/consts/mpich.jl) +for MPICH-compatible implementations. To create your own ABI file, it is +advisable to start with an existing constants file (e.g., for MPICH) and then +adapt each entry to the contents of your MPI implementations's `mpi.h` C header +file. + +For example, if your `mpi.h` header file contains something like +```c +typedef unsigned int MPI_Request; +enum { + MPI_REQUEST_NULL = 0 +}; + +#define MPI_ARGV_NULL ((char**) NULL) +``` +you need to put the corresponding entries in your ABI file `abi_source_file.jl`: +```julia +const MPI_Request = Cuint +@const_ref MPI_REQUEST_NULL MPI_Request 0 + +@const_ref MPI_ARGV_NULL Ptr{Cvoid} C_NULL +``` +As you can see, the syntax of such a Julia ABI file is non-trivial, thus the +recommendation to start with an existing ABI file. +It is further advisable to always use the corresponding Julia alias for +standard C types, e.g., `Cuint` for `unsigned int` or `Clonglong` for `long +long`. +Please note that sometimes information is also stored in ancillary header files (e.g., +`mpi_types.h` or `mpi_ext.h`). + +You can then use [`MPIPreferences.use_system_binary`](@ref) to configure MPI.jl +to use your custom file by providing the path via the `abi_source_file` keyword +argument, e.g., +```shell +julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary(; abi_source_file="path/to/file.jl)' +``` +You need to restart Julia for the change to take effect. + + ## Using an alternative JLL-provided MPI library The following MPI implementations are provided as JLL packages and automatically obtained when installing MPI.jl: diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl index 499ee8e3d..24acbb5ab 100644 --- a/lib/MPIPreferences/src/MPIPreferences.jl +++ b/lib/MPIPreferences/src/MPIPreferences.jl @@ -26,6 +26,7 @@ The ABI of the currently selected binary. Supported values are: - `"MicrosoftMPI"`: Microsoft MPI - `"MPItrampoline"`: MPItrampoline - `"HPE MPT"`: HPE MPT +- `"custom"`: ABI is defined via custom ABI source file """ const abi = if binary == "system" @load_preference("abi") @@ -45,6 +46,15 @@ end include("system.jl") end +""" + MPIPreferences.abi_source_file :: Union{String, Nothing} + +Path to a custom ABI source file holding constants for an MPI implementation not supported +out-of-the-box by MPI.jl. Will only be used if MPI binary is set to `"system"` and the ABI is set to +`"custom"` in [`use_system_binary`](@ref). +""" +const abi_source_file = @load_preference("abi_source_file") + """ MPIPreferences.use_jll_binary(binary; export_prefs=false, force=true) @@ -65,6 +75,7 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j "binary" => binary, "libmpi" => nothing, "abi" => nothing, + "abi_source_file" => nothing, "mpiexec" => nothing; export_prefs=export_prefs, force=force @@ -89,6 +100,7 @@ end library_names = ["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpitrampoline"], mpiexec = "mpiexec", abi = nothing, + abi_source_file = nothing, export_prefs = false, force = true) @@ -110,6 +122,11 @@ Options: - `abi`: the ABI of the MPI library. By default this is determined automatically using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported values. +- `abi_source_file`: the ABI file for the MPI library. By default, for ABIs supported by MPI.jl, the + corresponding ABI file is loaded automatically based on the value of `abi`. This argument allows + one to override the automatic selection, e.g., to provide an ABI file for an MPI ABI unknown to + MPI.jl. If specifying an ABI file, `abi` must not be set simultaneously. + - `export_prefs`: if `true`, the preferences into the `Project.toml` instead of `LocalPreferences.toml`. - `force`: if `true`, the preferences are set even if they are already set. @@ -117,7 +134,8 @@ Options: function use_system_binary(; library_names=["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpitrampoline"], mpiexec="mpiexec", - abi=nothing, + abi_source_file=nothing, + abi=isnothing(abi_source_file) ? nothing : "custom", export_prefs=false, force=true, ) @@ -132,6 +150,12 @@ function use_system_binary(; if isnothing(abi) abi = identify_abi(libmpi) end + if !isnothing(abi_source_file) + if abi != "custom" + error("`abi` must be set to `\"custom\"` when using ABI source file") + end + abi_source_file = abspath(abi_source_file) + end if mpiexec isa Cmd mpiexec = collect(mpiexec) end @@ -139,12 +163,13 @@ function use_system_binary(; "binary" => binary, "libmpi" => libmpi, "abi" => abi, + "abi_source_file" => abi_source_file, "mpiexec" => mpiexec, export_prefs=export_prefs, force=force ) - @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary libmpi abi mpiexec + @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary libmpi abi abi_source_file mpiexec if VERSION <= v"1.6.5" || VERSION == v"1.7.0" @warn """ diff --git a/src/consts/consts.jl b/src/consts/consts.jl index e1a98b2b3..485a4878c 100644 --- a/src/consts/consts.jl +++ b/src/consts/consts.jl @@ -27,6 +27,8 @@ macro const_ref(name, T, expr) :(const $(esc(name)) = Ref{$T}()) end +# Select package-provided ABI source file based on the value of the MPI ABI. If ABI is "custom", +# use custom ABI source file provided by user @static if MPIPreferences.abi == "MPICH" include("mpich.jl") elseif MPIPreferences.abi == "OpenMPI" @@ -37,6 +39,8 @@ elseif MPIPreferences.abi == "MPItrampoline" include("mpitrampoline.jl") elseif MPIPreferences.abi == "HPE MPT" include("mpt.jl") +elseif MPIPreferences.abi == "custom" + include(MPIPreferences.abi_source_file) else error("Unknown MPI ABI $(MPIPreferences.abi)") end