refactor: restore double precision executables (#24)

* make dp versions of mf2005, mfusg, mflgr, and mfnwt
* uses new --double argument for pymake make-program
* add modflow-devtools to requirements.txt, use ostags module
* use develop branches of pymake and devtools in requirements.txt
* rename scripts/build_executables.py -> scripts/build_programs.py
* test mf6 built by pymake not by meson (needed for mf5to6) in integration.yml
* check that test distribution zipfile exists in integration.yml
* update .gitignore
* format with black

.github/workflows/integration.yml

@@ -48,41 +48,62 @@ jobs:
         with:
           compiler: intel-classic
           version: 2021.7
-
+
+      # only necessary because we need mf5to6 for mf6 autotests
       - name: Build modflow6
         working-directory: modflow6
         run: |
           meson setup builddir --prefix=$(pwd) --libdir=bin -Ddebug=false
           meson install -C builddir
-
-      - name: Show meson build log
-        if: failure()
-        working-directory: modflow6
-        run: cat builddir/meson-logs/meson-log.txt
       
-      - name: Unit test modflow6
-        working-directory: modflow6
-        run: meson test --verbose --no-rebuild -C builddir
+      - name: Build programs
+        run: |
+          # build programs
+          mkdir programs
+          python executables/scripts/build_programs.py -p programs
 
-      - name: Update FloPy
-        working-directory: modflow6/autotest
-        run: python update_flopy.py
+          # move programs where mf6 autotests expect them
+          mkdir modflow6/bin/downloaded
+          cp programs/* modflow6/bin/downloaded
 
-      - name: Get executables
-        working-directory: modflow6/autotest
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        run: pytest -v -s get_exes.py
+          # move mf6 binaries to top-level bindir in mf6 repo
+          if [[ "$RUNNER_OS" == "Windows" ]]; then
+            eext=".exe"
+            oext=".dll"
+          elif [[ "$RUNNER_OS" == "Linux" ]]; then
+            eext=""
+            oext=".so"
+          else
+            eext=""
+            oext=".dylib"
+          fi
+          cp "programs/mf6$eext" modflow6/bin
+          cp "programs/libmf6$oext" modflow6/bin
+          cp "programs/zbud6$eext" modflow6/bin
+
+          # set execute permissions
+          if [[ "$RUNNER_OS" != "Windows" ]]; then
+            sudo chmod +x modflow6/bin/*
+            sudo chmod +x modflow6/bin/downloaded/*
+          fi
+      
+      - name: Upload programs
+        uses: actions/upload-artifact@v3
+        with:
+          name: programs
+          path: programs.zip
 
-      - name: Build executables
-        working-directory: executables/scripts
-        run: python build_executables.py -p ../../modflow6/bin/downloaded
-
-      - name: Set executable permission
-        if: runner.os != 'Windows'
-        working-directory: modflow6/bin/downloaded
-        run: sudo chmod +x *
+      - name: Upload metadata
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v3
+        with:
+          name: metadata
+          path: |
+            ./code.json
+            ./code.md
 
       - name: Test modflow6
         working-directory: modflow6/autotest
-        run: pytest -v -n auto --durations 0
+        run: |
+          python update_flopy.py
+          pytest -v -n auto -k "not gwe" -m "not developmode" --durations 0

.github/workflows/release.yml

@@ -54,7 +54,7 @@ jobs:
           pip list
 
       - name: Build executables
-        run: python scripts/build_executables.py
+        run: python scripts/build_programs.py
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v3

.gitignore

@@ -57,6 +57,9 @@ target/
 # PyCharm
 .idea/
 
-# files in the examples directory
+# Local testing folders
 temp/
+bin/
 
+# Python virtual environments
+venv/

requirements.txt

@@ -1,2 +1,3 @@
 wheel
-https://github.com/modflowpy/pymake/zipball/master
+https://github.com/modflowpy/pymake/zipball/develop
+https://github.com/MODFLOW-USGS/modflow-devtools/zipball/develop

scripts/build_executables.py → scripts/build_programs.py

@@ -4,22 +4,19 @@
 import textwrap
 from pathlib import Path
 
+from modflow_devtools.ostags import get_modflow_ostag
+
 DEFAULT_RETRIES = 3
+DBL_PREC_PROGRAMS = ["mf2005", "mflgr", "mfnwt", "mfusg"]
 
 
-def get_ostag() -> str:
-    """Determine operating system tag from sys.platform."""
-    if sys.platform.startswith("linux"):
-        return "linux"
-    elif sys.platform.startswith("win"):
-        return "win64"
-    elif sys.platform.startswith("darwin"):
-        return "mac"
-    raise ValueError(f"platform {sys.platform!r} not supported")
+def get_fc() -> str:
+    """Determine Intel Fortran compiler to use based on the current platform."""
+    return "ifort"
 
 
 def get_cc() -> str:
-    """Determine operating system tag from sys.platform."""
+    """Determine Intel C compiler to use based on the current platform."""
     if sys.platform.startswith("linux"):
         return "icc"
     elif sys.platform.startswith("win"):
@@ -46,7 +43,7 @@ def run_cmd(args) -> bool:
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=textwrap.dedent(
             """\
-            Build MODFLOW-releated executables, shared libraries and metadata files with pymake.
+            Build MODFLOW-related executables, shared libraries and metadata files with pymake.
             """
         ),
     )
@@ -60,7 +57,7 @@ def run_cmd(args) -> bool:
         "-p",
         "--path",
         required=False,
-        default=get_ostag(),
+        default=get_modflow_ostag(),
         help="Path to create built binaries and metadata files",
     )
     parser.add_argument(
@@ -70,40 +67,54 @@ def run_cmd(args) -> bool:
         required=False,
         default=DEFAULT_RETRIES,
         help="Number of times to retry a failed build",
-
     )
     args = parser.parse_args()
-
-    # whether to recreate existing binaries
     keep = bool(args.keep)
-
-    # output path
     path = Path(args.path)
     path.mkdir(parents=True, exist_ok=True)
-
-    # number of retries
+    zip_path = Path(path).with_suffix(".zip")
+    dp_programs = ",".join(DBL_PREC_PROGRAMS)
     retries = args.retries
-
-    # C compiler
+    fc = get_fc()
     cc = get_cc()
 
-    # create code.json
-    if not run_cmd([
-        "make-code-json",
-        "-f",
-        str(path / "code.json"),
-        "--verbose",
-    ]):
-        raise RuntimeError(f"could not make code.json")
-
-    # build binaries
-    build_args = [
-        "make-program", ":",
-        f"--appdir={path}",
-        "-fc=ifort", f"-cc={cc}",
-        f"--zip={path}.zip",
-    ]
-    if keep:
-        build_args.append("--keep")
-    if not run_cmd(build_args):
-        raise RuntimeError("could not build binaries")
+    assert run_cmd(
+        [
+            "make-code-json",
+            "-f",
+            str(path / "code.json"),
+            "--verbose",
+        ]
+    ), "could not make code.json"
+    assert run_cmd(
+        [
+            "make-program",
+            ":",
+            "--appdir",
+            path,
+            "-fc",
+            fc,
+            "-cc",
+            cc,
+            "--zip",
+            f"{path}.zip",
+        ]
+        + (["--keep"] if keep else [])
+    ), "could not build default precision binaries"
+    assert run_cmd(
+        [
+            "make-program",
+            dp_programs,
+            "--appdir",
+            path,
+            "--double",
+            "--keep",
+            "-fc",
+            fc,
+            "-cc",
+            cc,
+            "--zip",
+            str(zip_path),
+        ]
+    ), f"could not build double precision binaries: {dp_programs}"
+    assert zip_path.is_file(), "could not build distribution zipfile"