From 33d7e9b6dea9362accbe98eb34f5a8e1984c1833 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Sat, 7 Jan 2023 12:26:30 -0500 Subject: [PATCH] chore(tests): add large model test marker to pytest.ini --- DEVELOPER.md | 34 ++++++++++++---- autotest/pytest.ini | 8 ++-- autotest/targets.py | 98 --------------------------------------------- 3 files changed, 31 insertions(+), 109 deletions(-) delete mode 100644 autotest/targets.py diff --git a/DEVELOPER.md b/DEVELOPER.md index 976623e3a6b..3385e69a110 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -30,7 +30,6 @@ This document describes how to set up a development environment to modify, build - [Building](#building) - [Testing](#testing) - [Configuring a test environment](#configuring-a-test-environment) -- [Testing](#testing-1) - [Building development binaries](#building-development-binaries) - [Rebuilding and installing release binaries](#rebuilding-and-installing-release-binaries) - [Updating `flopy` plugins](#updating-flopy-plugins) @@ -41,6 +40,7 @@ This document describes how to set up a development environment to modify, build - [Running Tests](#running-tests) - [Selecting tests with markers](#selecting-tests-with-markers) - [External model tests](#external-model-tests) + - [Writing tests](#writing-tests) @@ -237,7 +237,6 @@ MODFLOW 6 tests are driven with [`pytest`](https://docs.pytest.org/en/7.1.x/), w A few tasks must be completed before running tests: -## Testing - build local MODFLOW 6 development version - rebuild the last MODFLOW 6 release - install additional executables @@ -362,6 +361,8 @@ pytest -v -n auto Markers can be used to select subsets of tests. Markers provided in `pytest.ini` include: - `slow`: tests that take longer than a few seconds to complete +- `repo`: tests that require external model repositories +- `large`: tests using large models (from the `modflow6-examples` and `modflow6-largetestmodels` repos) - `regression`: tests comparing results from multiple versions Markers can be used with the `-m ` option, and can be applied in boolean combinations with `and`, `or` and `not`. For instance, to run fast tests in parallel, excluding regression tests: @@ -386,18 +387,37 @@ Tests using models from external repositories can be selected with the `repo` ma pytest -v -n auto -m "repo" ``` -The test scripts can also be run independently: +The `large` marker is a subset of the `repo` marker. To test models excluded from commit-triggered CI and only run on GitHub Actions nightly: + +```shell +pytest -v -n auto -m "large" +``` + +Test scripts for external model repositories can also be run independently: ```shell -# Run MODFLOW 6 test models +# MODFLOW 6 test models pytest -v -n auto test_z01_testmodels_mf6.py -# Run MODFLOW 5 to 6 conversion test models +# MODFLOW 5 to 6 conversion test models pytest -v -n auto test_z02_testmodels_mf5to6.py -# Run example models +# models from modflow6-examples repo pytest -v -n auto test_z03_examples.py -# Run large test models +# models from modflow6-largetestmodels repo pytest -v -n auto test_z03_largetestmodels.py ``` + +#### Writing tests + +Tests should ideally follow a few conventions for easier maintenance: + +- Use temporary directory fixtures. Tests which write to disk should use `pytest`'s built-in `tmp_path` fixtures or one of the [keepable temporary directory fixtures from `modflow-devtools`](https://modflow-devtools.readthedocs.io/en/latest/md/fixtures.html#keepable-temporary-directories). This prevents tests from polluting one another's state. + +- Use markers for convenient (de-)selection: + - `@pytest.mark.slow` if the test doesn't complete in a few seconds (this preserves the ability to quickly [`--smoke` test](https://modflow-devtools.readthedocs.io/en/latest/md/markers.html#smoke-testing) + - `@pytest.mark.repo` if the test relies on external model repositories + - `@pytest.mark.regression` if the test compares results from different versions + +The test suite must pass before code can be merged, so be sure it passes locally before opening a PR. diff --git a/autotest/pytest.ini b/autotest/pytest.ini index 30cd6949ec3..013c63a54ff 100644 --- a/autotest/pytest.ini +++ b/autotest/pytest.ini @@ -10,7 +10,7 @@ markers = developmode: tests that should only run with IDEVELOPMODE = 1 gwf: tests for groundwater flow models gwt: tests for groundwater transport models - ats: tests for adaptive time step utility plugin - aux: tests for auxiliary variables plugin - lak: tests for flow lake plugin - maw: tests for multi-aquifer well plugin + ats: tests for adaptive time step package + aux: tests for auxiliary variables + lak: tests for lake package + maw: tests for multi-aquifer well package diff --git a/autotest/targets.py b/autotest/targets.py deleted file mode 100644 index e2c9ff54075..00000000000 --- a/autotest/targets.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import subprocess -import sys - -import flopy - - -def target_pth(target, pth): - exe_exists = flopy.which(target, path=pth) - # if target does not exist in specified path determine if it - # exists anywhere in the path - if exe_exists is None: - exe_exists = flopy.which(target) - if exe_exists is None: - exe_exists = os.path.abspath(os.path.join(pth, target)) - raise Exception(f"{exe_exists} does not exist or is not executable.") - return os.path.abspath(exe_exists) - - -target_ext = "" -target_so = ".so" -sysinfo = sys.platform.lower() -if sysinfo.lower() == "win32": - target_ext = ".exe" - target_so = ".dll" -elif sysinfo.lower() == "darwin": - target_so = ".dylib" - -# paths to executables for previous versions of MODFLOW -downloaded_bindir = os.path.join("..", "bin", "downloaded") -rebuilt_bindir = os.path.join("..", "bin", "rebuilt") - -# paths to MODFLOW 6 executable, source files, and example files -bindir = os.path.join("..", "bin") - -# create dictionary of valid executable targets for regression tests -target_dict = {} - -target = target_pth(f"mf2005dbl{target_ext}", downloaded_bindir) -target_dict["mf2005"] = target -target = target_pth(f"mfnwtdbl{target_ext}", downloaded_bindir) -target_dict["mfnwt"] = target -target = target_pth(f"mfusgdbl{target_ext}", downloaded_bindir) -target_dict["mfusg"] = target -target = target_pth(f"mflgrdbl{target_ext}", downloaded_bindir) -target_dict["mflgr"] = target -target = target_pth(f"mf2005{target_ext}", downloaded_bindir) -target_dict["mf2005s"] = target -target = target_pth(f"mt3dms{target_ext}", downloaded_bindir) -target_dict["mt3dms"] = target -target = target_pth(f"mf6{target_ext}", rebuilt_bindir) -target_dict["mf6_regression"] = target - -# create MODFLOW 6 target name and add to dictionary -program = f"mf6{target_ext}" -target = os.path.join(bindir, program) -target_dict["mf6"] = target - -# create MODFLOW 6 so/dll target name -tprog = f"libmf6{target_so}" -ttarg = os.path.join(bindir, tprog) -target_dict["libmf6"] = ttarg - -# add MODFLOW 5 to 6 converter to dictionary of valid executable targets -tprog = f"mf5to6{target_ext}" -ttarg = os.path.join(bindir, tprog) -target_dict["mf5to6"] = ttarg - -# add Zonebudget for 6 to dictionary of valid executable targets -tprog = f"zbud6{target_ext}" -ttarg = os.path.join(bindir, tprog) -target_dict["zbud6"] = ttarg - - -def run_exe(argv, ws="."): - buff = [] - proc = subprocess.Popen( - argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=ws - ) - result, error = proc.communicate() - if result is not None: - c = result.decode("utf-8") - c = c.rstrip("\r\n") - print(f"{c}") - buff.append(c) - - return proc.returncode, buff - - -def get_mf6_version(version="mf6"): - """Function to get MODFLOW 6 version number""" - exe = target_dict[version] - return_code, buff = run_exe((exe, "-v")) - if return_code == 0: - version = buff[0].split()[1] - else: - version = None - return version