Allow one to specify an ABI file via MPIPreferences

docs/src/configuration.md

@@ -35,6 +35,9 @@ standard or later. The following MPI implementations should work out-of-the-box
 - [Fujitsu MPI](https://www.fujitsu.com/global/about/resources/publications/technicalreview/2020-03/article07.html#cap-03)
 - [HPE MPT/HMPT](https://support.hpe.com/hpesc/public/docDisplay?docLocale=en_US&docId=a00105727en_us)
 
+If you are using an MPI implementation that is not ABI-compatible with any one
+of these, please read the section on [Supporting an unknown ABI](@ref) below.
+
 ### Configuration
 
 Run `MPIPreferences.use_system_binary()`. This will attempt to locate and to identify any available MPI implementation, and create a file called `LocalPreferences.toml` adjacent to the current `Project.toml`.
@@ -100,6 +103,50 @@ Preferences are merged across the Julia load path, such that it is feasible to p
    that will take precedent by modifying the local `Project.toml` or by providing a
    `LocalPreferences.toml` file.
 
+
+### Supporting an unknown ABI
+If you want to use an MPI implementation not officially supported by MPI.jl, you
+need to create your own ABI file with all relevant MPI constants. The files for supported
+ABIs are stored in the `src/consts/` folder, e.g.,
+[`mpich.jl`](https://github.com/JuliaParallel/MPI.jl/blob/master/src/consts/mpich.jl)
+for MPICH-compatible implementations. To create your own ABI file, it is
+advisable to start with an existing constants file (e.g., for MPICH) and then
+adapt each entry to the contents of your MPI implementations's `mpi.h` C header
+file.
+
+For example, if your `mpi.h` header file contains something like
+```c
+typedef unsigned int MPI_Request;
+enum {
+  MPI_REQUEST_NULL  = 0
+};
+
+#define MPI_ARGV_NULL ((char**) NULL)
+```
+you need to put the corresponding entries in your ABI file `abi_source_file.jl`:
+```julia
+const MPI_Request = Cuint
+@const_ref MPI_REQUEST_NULL MPI_Request 0
+
+@const_ref MPI_ARGV_NULL Ptr{Cvoid} C_NULL
+```
+As you can see, the syntax of such a Julia ABI file is non-trivial, thus the
+recommendation to start with an existing ABI file.
+It is further advisable to always use the corresponding Julia alias for
+standard C types, e.g., `Cuint` for `unsigned int` or `Clonglong` for `long
+long`.
+Please note that sometimes information is also stored in ancillary header files (e.g.,
+`mpi_types.h` or `mpi_ext.h`).
+
+You can then use [`MPIPreferences.use_system_binary`](@ref) to configure MPI.jl
+to use your custom file by providing the path via the `abi_source_file` keyword
+argument, e.g.,
+```shell
+julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary(; abi_source_file="path/to/file.jl)'
+```
+You need to restart Julia for the change to take effect.
+
+
 ## Using an alternative JLL-provided MPI library
 
 The following MPI implementations are provided as JLL packages and automatically obtained when installing MPI.jl:

lib/MPIPreferences/src/MPIPreferences.jl

@@ -26,6 +26,7 @@ The ABI of the currently selected binary. Supported values are:
 - `"MicrosoftMPI"`: Microsoft MPI
 - `"MPItrampoline"`: MPItrampoline
 - `"HPE MPT"`: HPE MPT
+- `"custom"`: ABI is defined via custom ABI source file
 """
 const abi = if binary == "system"
     @load_preference("abi")
@@ -45,6 +46,15 @@ end
     include("system.jl")
 end
 
+"""
+    MPIPreferences.abi_source_file :: Union{String, Nothing}
+
+Path to a custom ABI source file holding constants for an MPI implementation not supported
+out-of-the-box by MPI.jl. Will only be used if MPI binary is set to `"system"` and the ABI is set to
+`"custom"` in [`use_system_binary`](@ref).
+"""
+const abi_source_file = @load_preference("abi_source_file")
+
 """
     MPIPreferences.use_jll_binary(binary; export_prefs=false, force=true)
 
@@ -65,6 +75,7 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
         "binary" => binary,
         "libmpi" => nothing,
         "abi" => nothing,
+        "abi_source_file" => nothing,
         "mpiexec" => nothing;
         export_prefs=export_prefs,
         force=force
@@ -89,6 +100,7 @@ end
         library_names = ["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpitrampoline"],
         mpiexec = "mpiexec",
         abi = nothing,
+        abi_source_file = nothing,
         export_prefs = false,
         force = true)
 
@@ -110,14 +122,20 @@ Options:
 - `abi`: the ABI of the MPI library. By default this is determined automatically
   using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported values.
 
+- `abi_source_file`: the ABI file for the MPI library. By default, for ABIs supported by MPI.jl, the
+  corresponding ABI file is loaded automatically based on the value of `abi`. This argument allows
+  one to override the automatic selection, e.g., to provide an ABI file for an MPI ABI unknown to
+  MPI.jl. If specifying an ABI file, `abi` must not be set simultaneously.
+
 - `export_prefs`: if `true`, the preferences into the `Project.toml` instead of `LocalPreferences.toml`.
 
 - `force`: if `true`, the preferences are set even if they are already set.
 """
 function use_system_binary(;
         library_names=["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpitrampoline"],
         mpiexec="mpiexec",
-        abi=nothing,
+        abi_source_file=nothing,
+        abi=isnothing(abi_source_file) ? nothing : "custom",
         export_prefs=false,
         force=true,
     )
@@ -132,19 +150,26 @@ function use_system_binary(;
     if isnothing(abi)
         abi = identify_abi(libmpi)
     end
+    if !isnothing(abi_source_file)
+        if abi != "custom"
+            error("`abi` must be set to `\"custom\"` when using ABI source file")
+        end
+        abi_source_file = abspath(abi_source_file)
+    end
     if mpiexec isa Cmd
         mpiexec = collect(mpiexec)
     end
     set_preferences!(MPIPreferences,
         "binary" => binary,
         "libmpi" => libmpi,
         "abi" => abi,
+        "abi_source_file" => abi_source_file,
         "mpiexec" => mpiexec,
         export_prefs=export_prefs,
         force=force
     )
 
-    @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary libmpi abi mpiexec
+    @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary libmpi abi abi_source_file mpiexec
 
     if VERSION <= v"1.6.5" || VERSION == v"1.7.0"
         @warn """

src/consts/consts.jl

@@ -27,6 +27,8 @@ macro const_ref(name, T, expr)
     :(const $(esc(name)) = Ref{$T}())
 end
 
+# Select package-provided ABI source file based on the value of the MPI ABI. If ABI is "custom",
+# use custom ABI source file provided by user
 @static if MPIPreferences.abi == "MPICH"
     include("mpich.jl")
 elseif MPIPreferences.abi == "OpenMPI"
@@ -37,6 +39,8 @@ elseif MPIPreferences.abi == "MPItrampoline"
     include("mpitrampoline.jl")
 elseif MPIPreferences.abi == "HPE MPT"
     include("mpt.jl")
+elseif MPIPreferences.abi == "custom"
+    include(MPIPreferences.abi_source_file)
 else
     error("Unknown MPI ABI $(MPIPreferences.abi)")
 end