Merge pull request #19 from aurora-multiphysics/3-drive-ascot5-runs

Add ability to drive sequential ASCOT5 runs
aurora-multiphysics · Mar 11, 2022 · fb653bb · fb653bb
2 parents 28242bd + af42347
commit fb653bb
Show file tree

Hide file tree

Showing 41 changed files with 1,300 additions and 118 deletions.
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
@@ -4,6 +4,7 @@
             "name": "MOOSE-Linux",
             "includePath": [
                 "${workspaceFolder}",
+                "${workspaceFolder}/ascot5/include",
                 "${workspaceFolder}/build/header_symlinks",
                 "${workspaceFolder}/unit/build/header_symlinks",
                 "${MOOSE_DIR}/framework/build/header_symlinks",
@@ -24,6 +25,7 @@
             "browse": {
                 "path": [
                     "${workspaceFolder}",
+                    "${workspaceFolder}/ascot5/include",
                     "${workspaceFolder}/build/header_symlinks",
                     "${workspaceFolder}/unit/build/header_symlinks",
                     "${MOOSE_DIR}/framework/build/header_symlinks",

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -38,14 +38,14 @@
             }
         },
         {
-            "name": "(lldb) Unit Test Launch",
+            "name": "(gdb) Unit Test Launch",
             "type": "cppdbg",
             "request": "launch",
-            "program": "${workspaceFolder}/unit/phaethon-unit-dbg",
+            "program": "${workspaceFolder}/unit/phaethon-unit-devel",
             "stopAtEntry": false,
-            "cwd": "${workspaceFolder}",
-            "environment": [],
-            "externalConsole": true,
+            "cwd": "${workspaceFolder}/unit",
+            "environment": [{"name": "ASAN_OPTIONS", "value": "alloc_dealloc_mismatch=0"}],
+            "externalConsole": false,
             "MIMode": "gdb",
             "logging": {
                 "trace": true,

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -14,10 +14,70 @@
         "sstream": "cpp",
         "streambuf": "cpp",
         "vector": "cpp",
-        "*.csv": "cpp"
+        "*.csv": "cpp",
+        "array": "cpp",
+        "atomic": "cpp",
+        "strstream": "cpp",
+        "bit": "cpp",
+        "bitset": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "codecvt": "cpp",
+        "complex": "cpp",
+        "condition_variable": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "forward_list": "cpp",
+        "list": "cpp",
+        "map": "cpp",
+        "set": "cpp",
+        "unordered_map": "cpp",
+        "unordered_set": "cpp",
+        "exception": "cpp",
+        "algorithm": "cpp",
+        "functional": "cpp",
+        "iterator": "cpp",
+        "memory": "cpp",
+        "memory_resource": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "random": "cpp",
+        "regex": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "future": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "iostream": "cpp",
+        "mutex": "cpp",
+        "new": "cpp",
+        "ostream": "cpp",
+        "stdexcept": "cpp",
+        "thread": "cpp",
+        "cfenv": "cpp",
+        "cinttypes": "cpp",
+        "typeinfo": "cpp",
+        "valarray": "cpp",
+        "nonlinearoptimization": "cpp",
+        "tensor": "cpp"
     },
     "python.linting.flake8Enabled": true,
     "python.linting.enabled": true,
     "restructuredtext.languageServer.disabled": true,
-    "python.pythonPath": "/usr/bin/python"
+    "python.pythonPath": "/usr/bin/python",
+    "esbonio.server.enabled": false,
+    "esbonio.sphinx.buildDir": "/home/mbluteau/work/projects/moose_fi_app/code/phaethon/build"
 }
diff --git a/Makefile b/Makefile
@@ -49,10 +49,6 @@ XFEM                        := no
 include $(MOOSE_DIR)/modules/modules.mk
 ###############################################################################
 
-# Additional libraries for HDF5 support
-ADDITIONAL_INCLUDES := -I/usr/include/hdf5/serial
-ADDITIONAL_LIBS     := -lhdf5_hl_cpp -lhdf5_cpp -lhdf5_serial_hl -lhdf5_serial
-
 # dep apps
 APPLICATION_DIR    := $(CURDIR)
 APPLICATION_NAME   := phaethon

diff --git a/ascot5 b/ascot5
diff --git a/doc/content/releases.md b/doc/content/releases.md
@@ -2,13 +2,12 @@
 
 1. Choose a version number. Look at previous releases and use [semantic versioning](https://semver.org/) to determine what the next number should be.
 2. Update the `CHANGELOG` at the top level of the repository for this version, summarising any major changes made since the last release. Use **Fixed**, **Added**, and **Changed** sections.
-3. Commit CHANGELOG.
-3. Make sure you are on the main branch and there are no uncommitted changes.
-4. Mark release with an empty git commit: `git commit --allow-empty -m "REL: vX.Y.Z"`
+3. Make sure you are on the main branch and there are no other uncommitted changes.
+4. Commit the changelog and mark as a release: `git commit -m "REL: vX.Y.Z"`
 5. Tag the commit: `git tag -a vX.Y.Z` and then enter brief description.
 6. Push the tag and commit to remote: `git push origin main && git push origin vX.Y.Z`
 7. Build the Docker image: `docker build --file Dockerfile --target phaethon --tag phaethon:vX.Y.Z .`
-8. Save the Docker image to a compressed tar archive: `docker save phaethon:vX.Y.Z | gzip > phaethon_vX.Y.Z.tar.gz`
+8. Save the Docker image to a compressed tar archive: `docker save phaethon:vX.Y.Z | gzip > phaethon-ubuntu-vX_Y_Z.tar.gz`
 9. Upload tar archive to [STEP OneDrive](https://ukaeauk.sharepoint.com/:f:/s/STEP_DigitalEnablers_B_3_2/Ei608KfIoLBNlSjtyUf1XMEB-V8nUKx0iFX5p07Z7VE9xw?e=eNWutU)
 
 TODO step 4 onwards could be automated by a script or Makefile with the version number as the input
diff --git a/docker/Dockerfile-phaethon b/docker/Dockerfile-phaethon
@@ -4,11 +4,11 @@
 # TODO should probably move away from using submodules. It means the version
 # information of dependencies is scattered, and it would be better to centralise
 # it here.
-ARG MOOSE_BASE=moose-ubuntu-dev MOOSE_VER=2021-09-15
+ARG MOOSE_BASE=moose-ubuntu-dev MOOSE_VER=2021-09-15 USER=dev
 FROM ${MOOSE_BASE}:${MOOSE_VER} AS ascot5-moose-ubuntu
 
 # Basic environment
-ARG USER=dev
+ARG USER
 WORKDIR $HOME
 # Set path to shared library for Python interface to use and other environment
 ENV LD_LIBRARY_PATH=$HOME/ascot5 EDITOR=vim PATH="${HOME}/.local/bin:${PATH}"
@@ -42,7 +42,7 @@ RUN python -m pip install --upgrade meshio[all] click
 FROM phaethon-deps as phaethon-dev
 
 # Some development utilities, including git-lfs
-RUN sudo apt-get -y install clang-format curl hdf5-tools
+RUN sudo apt-get -y install clang-format curl hdf5-tools gdb
 RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash && \
     sudo apt-get -y install git-lfs && \
     git lfs install
@@ -55,7 +55,10 @@ RUN python -m pip install --upgrade flake8 jupyter
 ##############################
 FROM phaethon-deps as phaethon
 
-WORKDIR /home/phaethon
+ARG USER
+WORKDIR $HOME/phaethon
 COPY ./ ./
-RUN git clean -dfx && make -j4 && make -C unit/ -j4 
-RUN cd unit && ./run_tests
+
+RUN sudo chown -R ${USER}:${USER} . && git clean -dfx \
+    && make -j4 && make -C unit/ -j4 
+RUN ./run_tests && cd unit && ./run_tests
diff --git a/include/problems/AscotProblem.h b/include/problems/AscotProblem.h
@@ -60,6 +60,79 @@ class AscotProblem : public ExternalProblem
    */
   static H5::Group getActiveEndstate(const H5::H5File & hdf5_file);
 
+  /**
+   * @brief Get the Marker Group object
+   *
+   * @param hdf5_file the ASCOT5 HDF5 file with input and output data.
+   * @return H5::Group the HDF5 file group object for the markers
+   */
+  static H5::Group getAscotH5Group(const H5::H5File & hdf5_file, const std::string & group_name);
+
+  /**
+   * @brief Get an Ascot HDF5 Data object
+   *
+   * @tparam T type of the data object
+   * @param endstate_group the group in which the data object sits
+   * @param field_name the name of the data object in the HDF5 file
+   * @return std::vector<T> the data object values
+   */
+  template <class T>
+  static std::vector<T> getAscotH5DataField(H5::Group & endstate_group,
+                                            const std::string & field_name);
+
+  /**
+   * @brief Set an Ascot HDF5 Data object
+   *
+   * @tparam T type of the data object
+   * @param endstate_group the group to which the data object will be written
+   * @param field_name the name of the data object in the HDF5 file
+   */
+  template <class T>
+  static void setAscotH5DataField(H5::Group & endstate_group, const std::string & field_name);
+
+  /**
+   * @brief Create and write a dataset to an HDF5 group
+   *
+   * @tparam T the type of data
+   * @param data the data to write to the HDF5 DataSet
+   * @param name the name of the data field in the HDF5 file
+   * @param data_space the HDF5 DataSpace for the DataSet
+   * @param group the HDF5 Group in which the DataSet will be created
+   */
+  template <class T>
+  static void createAndWriteDataset(const std::vector<T> & data,
+                                    const std::string & name,
+                                    const H5::DataSpace & data_space,
+                                    const H5::Group & group);
+
+  /**
+   * @brief Copy the ASCOT5 endstate data members to the marker group in the HDF5 file
+   *
+   * @param marker_group the marker group in the HDF5 file to which the endstate
+   *        is to be copied
+   */
+  void copyEndstate2MarkerGroup(const H5::H5File & hdf5_file);
+
+  /**
+   * @brief Get the Double type Ascot Endstate Variables from HDF5 File
+   *
+   * @param endstate_group the HDF5 file group where the endstate lives
+   * @return std::unordered_map<std::string, std::vector<double_t>> the variables of type double_t
+   * in the endstate
+   */
+  static std::unordered_map<std::string, std::vector<double_t>>
+  getAscotH5EndstateDouble(H5::Group & endstate_group);
+
+  /**
+   * @brief Get the Integer type Ascot Endstate Variables from HDF5 File
+   *
+   * @param endstate_group the HDF5 file group where the endstate lives
+   * @return std::unordered_map<std::string, std::vector<double_t>> the variables of type int64_t in
+   * the endstate
+   */
+  static std::unordered_map<std::string, std::vector<int64_t>>
+  getAscotH5EndstateInt(H5::Group & endstate_group);
+
   /**
    * @brief Get the indices of wall tiles that each particle has collided with
    *
@@ -110,12 +183,21 @@ class AscotProblem : public ExternalProblem
                                             std::vector<double_t> energies,
                                             std::vector<double_t> weights);
 
+  // Endstate variables required for restarting ASCOT5
+  std::unordered_map<std::string, std::vector<double_t>> endstate_data_double;
+  std::unordered_map<std::string, std::vector<int64_t>> endstate_data_int;
+
 private:
   /// The name of the AuxVariable to transfer to
   const VariableName & _sync_to_var_name;
   /// The Auxiliary system in which the heat flux values will be stored
   AuxiliarySystem & _problem_system;
   /// The HDF5 file that is both the ASCOT5 input and output
-  const H5std_string _ascot5_file_name;
+  const FileName & _ascot5_file_name;
   H5::H5File _ascot5_file;
+  /// Mapping for top-level group name to sub-group prefix for ASCOT5 HDF5 file
+  static const std::unordered_map<std::string, std::string> hdf5_group_prefix;
+  // The HDF5 dataset names that constitute the required endstate variables to restart ASCOT5
+  static const std::vector<std::string> endstate_fields_double;
+  static const std::vector<std::string> endstate_fields_int;
 };
diff --git a/phaethon.mk b/phaethon.mk
@@ -0,0 +1,21 @@
+## Some build configurations specific to Phaethon
+# This file is automatically included by $(FRAMEWORK_DIR)/app.mk so it doesn't
+# need to be included in this project's Makefile
+
+# Additional libraries for HDF5 support
+ifdef CONDA_DEFAULT_ENV
+ADDITIONAL_LIBS     += -lhdf5_hl_cpp -lhdf5_cpp -lhdf5_hl -lhdf5
+else
+ADDITIONAL_INCLUDES += -I/usr/include/hdf5/serial
+ADDITIONAL_LIBS     += -lhdf5_hl_cpp -lhdf5_cpp -lhdf5_serial_hl -lhdf5_serial
+endif
+
+# ASCOT5
+ASCOT5_DIR := $(APPLICATION_DIR)/ascot5
+ADDITIONAL_DEPEND_LIBS += libascot_main
+ADDITIONAL_LIBS += -Wl,-rpath=$(ASCOT5_DIR) -L$(ASCOT5_DIR) -lascot_main
+ADDITIONAL_INCLUDES += -I$(ASCOT5_DIR)/include
+ASCOT5_OPT := NOGIT=true CC=h5cc MPI=0 FLAGS=-foffload=disable
+
+libascot_main:
+	$(MAKE) $(ASCOT5_OPT) -C $(ASCOT5_DIR) libascot_main