Merge remote-tracking branch 'origin/develop' into stable

.github/workflows/ci-github-actions.yaml

@@ -0,0 +1,85 @@
+name: GitHub Actions CI
+
+on: 
+  push:
+    branches: 
+    - stable
+    - develop
+  pull_request:
+    branches: 
+    - develop
+
+jobs:
+
+  linux:
+    runs-on: ubuntu-latest
+    container: ${{ matrix.container }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_JOBNAME: ${{ matrix.jobname }}
+      GH_OS: Linux
+    strategy:
+      fail-fast: false
+      matrix:
+        jobname: [
+          ubuntu22-gcc,
+          ubuntu22-clang,
+        ]
+        include:
+        - jobname: ubuntu22-gcc
+          container: 
+            image: ghcr.io/philipfackler/xolotl-ci:ubuntu22-dev
+            options: -u root 
+        - jobname: ubuntu22-clang
+          container: 
+            image: ghcr.io/philipfackler/xolotl-ci:ubuntu22-dev
+            options: -u root
+
+    steps:
+    - name: Checkout Action
+      uses: actions/checkout@v1
+
+    - name: Configure
+      run: CI/scripts/run_step.sh configure
+
+    - name: Build
+      run: CI/scripts/run_step.sh build
+
+    - name: Test
+      run: CI/scripts/run_step.sh test
+
+
+  macos:
+    runs-on: macos-latest
+    env:
+      GH_YML_JOBNAME: ${{ matrix.jobname }}
+      GH_YML_OS: macOS
+
+    strategy:
+      fail-fast: false
+      matrix:
+        jobname: [
+          macos1015-xcode11 
+        ]
+
+    steps:
+    - name: Checkout Action
+      uses: actions/checkout@v1
+
+    - name: Setup Dependencies
+      run: |
+        brew install boost open-mpi hdf5-mpi make [email protected]
+        echo "PATH=/opt/homebrew/opt/[email protected]/libexec/bin:$PATH" >> $GITHUB_ENV
+
+    - name: Configure
+      run: |
+        export EXTRA_CMAKE_ARGS='-DXolotl_INCLUDE_RN_TPP_FILES=ON -DXolotl_ENABLE_TIMER_TESTS=OFF'
+        CI/scripts/run_step.sh configure
+      env:
+        PATH: ${{ env.PATH }}
+
+    - name: Build
+      run: CI/scripts/run_step.sh build
+
+    - name: Test
+      run: CI/scripts/run_step.sh test

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "external/plsm"]
 	path = external/plsm
 	url = https://github.com/ORNL-Fusion/plsm.git
+[submodule "external/petsc"]
+	path = external/petsc
+	url = https://gitlab.com/petsc/petsc.git

CI/docker/ubuntu22/Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies
+RUN apt-get update -y
+RUN apt-get upgrade -y
+RUN apt-get install -y \
+        python3 \
+        clang \
+        clang-tidy \
+        g++ \
+        git \
+        gdb \
+        make \
+        cmake \
+        cmake-curses-gui \
+        mpi-default-dev \
+        libblas-dev \
+        liblapack-dev \
+        libhdf5-mpi-dev \
+        libboost-filesystem-dev \
+        libboost-program-options-dev \
+        libboost-test-dev \
+        libboost-log-dev \
+        bzip2 \
+        sudo
+
+RUN useradd -mr -u 10000 builduser
+RUN adduser builduser sudo
+RUN echo "builduser:builduser" | chpasswd
+
+USER builduser
+WORKDIR /home/builduser

CI/scripts/run_step.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -e
+
+echo -e "\nCheck gcc and clang compilers\n"
+gcc --version
+clang --version
+
+function run_tests () {
+    cd ${GITHUB_WORKSPACE}/../build
+    ctest --output-on-failure --label-exclude xolotl.tests.system
+    local __xolotl_ret=$?
+    ./test/system/SystemTester -- -t
+    local __xolotl_sys_ret=$?
+    return $((__xolotl_ret + __xolotl_sys_ret))
+}
+
+case "$1" in 
+
+  configure)
+
+    git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+    case "${GH_JOBNAME}" in
+      *"clang"*)
+        export CC=clang
+        export CXX=clang++
+        export OMPI_CC=clang
+        export OMPI_CXX=clang++
+      ;;
+      *)
+      ;;
+    esac
+
+    cd ${GITHUB_WORKSPACE}/..
+
+    git clone https://gitlab.com/petsc/petsc.git -b v3.22.2 petsc
+    cd petsc
+    bash ${GITHUB_WORKSPACE}/../xolotl/scripts/build_petsc.sh \
+        --prefix=${GITHUB_WORKSPACE}/../install \
+        --skip-pull
+
+    cd ${GITHUB_WORKSPACE}/..
+    mkdir build
+    cd build
+
+    cmake \
+        -DCMAKE_PREFIX_PATH=${GITHUB_WORKSPACE}/../install \
+        ${EXTRA_CMAKE_ARGS} \
+        ${GITHUB_WORKSPACE}
+
+    ;;
+
+  build)
+    cd ${GITHUB_WORKSPACE}/../build
+    make -j4
+    ;;
+
+  test)
+    run_tests
+    ;;
+
+  *)
+    echo " Invalid step" "$1"
+    exit -1
+    ;;
+esac

CMake/BuildPETSc.cmake

@@ -0,0 +1,159 @@
+find_package(Git REQUIRED)
+
+message(STATUS "Setup PETSc")
+
+## Set directories
+set(__external_src_dir ${CMAKE_SOURCE_DIR}/external)
+set(__external_bin_dir ${CMAKE_BINARY_DIR}/external)
+
+set(__petsc_src_dir ${__external_src_dir}/petsc)
+set(__petsc_bin_dir ${__external_bin_dir}/petsc_build)
+
+execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${__petsc_bin_dir})
+execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${__petsc_bin_dir})
+
+if(NOT EXISTS ${__petsc_src_dir}/configure)
+    message(STATUS "    checkout")
+    execute_process(
+        COMMAND ${GIT_EXECUTABLE} submodule update --init external/petsc
+        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+        OUTPUT_FILE "${__external_bin_dir}/petsc_clone.out"
+        ERROR_FILE "${__external_bin_dir}/petsc_clone.out"
+    )
+endif()
+
+## Check for dependencies
+#### HDF5
+option(Xolotl_BUILD_HDF5 "Have the PETSc build system build HDF5" OFF)
+if(NOT Xolotl_BUILD_HDF5)
+    set(HDF5_PREFER_PARALLEL ON)
+    find_package(HDF5 QUIET)
+endif()
+#### Boost
+option(Xolotl_BUILD_BOOST "Have the PETSc build system build Boost" OFF)
+if(NOT Xolotl_BUILD_BOOST)
+    find_package(Boost QUIET COMPONENTS
+        log_setup
+        log
+        program_options
+    )
+endif()
+#### BLAS/LAPACK
+option(Xolotl_BUILD_LAPACK "Have the PETSc build system build LAPACK" OFF)
+if(NOT Xolotl_BUILD_LAPACK)
+    find_package(LAPACK QUIET)
+endif()
+
+## Set options
+set(__script_dir ${CMAKE_SOURCE_DIR}/scripts)
+set(__build_opts
+    --skip-pull
+    --prefix=${__external_bin_dir}/petsc_install
+)
+
+option(Xolotl_BUILD_PETSC_DEBUG
+    "Enable debugging symbols for petsc, kokkos, etc."
+    OFF
+)
+if(Xolotl_BUILD_PETSC_DEBUG)
+    list(APPEND __build_opts --debug)
+    message(STATUS "    - enable debugging")
+endif()
+
+option(Xolotl_ENABLE_CUDA "Enable CUDA backend for kokkos, etc." OFF)
+option(Xolotl_ENABLE_OPENMP "Enable OpenMP backend for kokkos" OFF)
+if(Xolotl_ENABLE_CUDA)
+    list(APPEND __build_opts --cuda)
+    message(STATUS "    - using CUDA backend")
+    if(Xolotl_CUDA_SM)
+        list(APPEND __build_opts --cuda-sm=${Xolotl_CUDA_SM})
+    else()
+        execute_process(
+            COMMAND nvidia-smi
+            OUTPUT_VARIABLE __smi_out
+            ERROR_VARIABLE __smi_out
+            RESULT_VARIABLE __smi_avail
+        )
+        if(NOT ${__smi_avail} EQUAL 0)
+            message(FATAL_ERROR "
+                Unable to determine CUDA SM version (compute capability).
+                Please provide this in Xolotl_CUDA_SM.
+                "
+            )
+        endif()
+    endif()
+elseif(Xolotl_ENABLE_OPENMP)
+    list(APPEND __build_opts --openmp)
+    message(STATUS "    - using OpenMP backend")
+else()
+    message(STATUS "    - using Serial backend")
+endif()
+
+if(Xolotl_KOKKOS_VERSION)
+    list(APPEND __build_opts --kokkos-version=${Xolotl_KOKKOS_VERSION})
+endif()
+
+## Include dependencies if necessary
+#### HDF5
+if(NOT HDF5_FOUND)
+    set(Xolotl_BUILD_HDF5 ON)
+endif()
+if(Xolotl_BUILD_HDF5)
+    list(APPEND __build_opts --get-hdf5)
+    message(STATUS "    - build HDF5")
+    set(HDF5_ROOT "${__external_bin_dir}/petsc_install")
+endif()
+#### Boost
+if(NOT Boost_FOUND)
+    set(Xolotl_BUILD_BOOST ON)
+endif()
+if(Xolotl_BUILD_BOOST)
+    list(APPEND __build_opts --get-boost)
+    message(STATUS "    - build Boost")
+    if(NOT Xolotl_BUILD_PETSC_DEBUG)
+        set(Boost_USE_DEBUG_RUNTIME OFF CACHE INTERNAL "")
+    endif()
+endif()
+#### LAPACK
+if(NOT LAPACK_FOUND)
+    set(Xolotl_BUILD_LAPACK ON)
+endif()
+if(Xolotl_BUILD_LAPACK)
+    list(APPEND __build_opts --get-lapack)
+    message(STATUS "    - build BLAS/LAPACK")
+endif()
+
+## Perform build
+set(__output_file "${__external_bin_dir}/petsc_build.out")
+message(STATUS "    build (for output, follow ${__output_file})")
+set(__command bash ${__script_dir}/build_petsc.sh ${__build_opts})
+if(Xolotl_BUILD_PETSC_DRY_RUN)
+    string(REPLACE ";" " " __command_str "${__command}")
+    message(STATUS "Script Command:")
+    message(STATUS "${__command_str}")
+    execute_process(
+        COMMAND ${__command} --dry-run
+        WORKING_DIRECTORY "${__petsc_src_dir}"
+    )
+    message(FATAL_ERROR "Exiting")
+endif()
+execute_process(
+    COMMAND ${__command}
+    WORKING_DIRECTORY "${__petsc_src_dir}"
+    OUTPUT_FILE "${__output_file}"
+    ERROR_FILE "${__output_file}"
+    RESULT_VARIABLE __build_ret
+)
+if(NOT ${__build_ret} EQUAL 0)
+    message(FATAL_ERROR "
+        Failed to build PETSc
+        See \"${__output_file}\"
+        "
+    )
+endif()
+
+list(APPEND CMAKE_PREFIX_PATH "${__external_bin_dir}/petsc_install")
+set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} CACHE PATH "" FORCE)
+
+## Don't build when re-running CMake unless the user specifies this again
+set(Xolotl_BUILD_PETSC OFF CACHE PATH "" FORCE)

CMake/FindPETSc.cmake

@@ -43,19 +43,23 @@ find_path(PETSCCONF_INCLUDE_DIR
     $ENV{PETSC_DIR}/$ENV{PETSC_ARCH}/include
 )
 
-get_filename_component(PETSC_LIB_DIR ${PETSC_LIBRARY} DIRECTORY)
-get_filename_component(PETSC_DIR ${PETSC_LIB_DIR} DIRECTORY)
-set(PETSc_DIR "${PETSC_DIR}" CACHE PATH "" FORCE)
-set(PETSc_LIBRARIES "${PETSC_LIBRARY}" CACHE FILEPATH "" FORCE)
+if(PETSC_LIBRARY)
+    get_filename_component(PETSC_LIB_DIR ${PETSC_LIBRARY} DIRECTORY)
+    get_filename_component(PETSC_DIR ${PETSC_LIB_DIR} DIRECTORY)
+    set(PETSc_DIR "${PETSC_DIR}" CACHE PATH "" FORCE)
+    set(PETSc_LIBRARIES "${PETSC_LIBRARY}" CACHE FILEPATH "" FORCE)
+endif()
 list(APPEND PETSC_INCLUDES "${PETSC_INCLUDE_DIR}" "${PETSCCONF_INCLUDE_DIR}")
 list(REMOVE_DUPLICATES PETSC_INCLUDES)
 set(PETSc_INCLUDES ${PETSC_INCLUDES} CACHE PATH "" FORCE)
 
-file(READ ${PETSCCONF_INCLUDE_DIR}/petscconf.h _petsc_conf)
-set(PETSc_USE_64BIT_INDICES FALSE)
-string(FIND ${_petsc_conf} "#define PETSC_USE_64BIT_INDICES 1" _id)
-if(${_id} GREATER -1)
-    set(PETSc_USE_64BIT_INDICES TRUE)
+if(PETSCCONF_INCLUDE_DIR)
+    file(READ ${PETSCCONF_INCLUDE_DIR}/petscconf.h _petsc_conf)
+    set(PETSc_USE_64BIT_INDICES FALSE)
+    string(FIND ${_petsc_conf} "#define PETSC_USE_64BIT_INDICES 1" _id)
+    if(${_id} GREATER -1)
+        set(PETSc_USE_64BIT_INDICES TRUE)
+    endif()
 endif()
 
 unset(PETSC_LIBRARY CACHE)