diff --git a/.github/workflows/call-pr-1-ci.yml b/.github/workflows/call-pr-1-ci.yml index a817c111..57dbf1ba 100644 --- a/.github/workflows/call-pr-1-ci.yml +++ b/.github/workflows/call-pr-1-ci.yml @@ -12,7 +12,6 @@ on: - tools/** - doc/** - .* - - metadata.yaml - README.md jobs: call: diff --git a/.github/workflows/pr-1-ci.yml b/.github/workflows/pr-1-ci.yml index 5a9f972f..1144ae59 100644 --- a/.github/workflows/pr-1-ci.yml +++ b/.github/workflows/pr-1-ci.yml @@ -1,6 +1,22 @@ name: PR Checks on: workflow_call: + inputs: + qa-pytest-markers: + type: string + required: false + default: config + description: List of markers for the pytest QA CI checks + qa-pytest-add-model-markers: + type: boolean + required: false + default: true + description: Whether to run model-specific tests + repro-pytest-markers: + type: string + required: false + default: checksum + description: List of markers for the pytest repro CI checks # Workflows that call this workflow use the following triggers: # pull_request: # branches: @@ -12,7 +28,6 @@ on: # - tools/** # - doc/** # - .* - # - metadata.yaml # - README.md jobs: commit-check: @@ -35,10 +50,15 @@ jobs: branch-check: name: PR Source Branch Check + # This check is used as a precursor to any repro-ci checks - which are only fired + # on dev-* -> release-* PRs. # This check is run to confirm that the source branch is of the form `dev-` # and the target branch is of the form `release-`. We are being especially # concerned with branch names because deployment to GitHub Environments can only - # be done on source branches with a certain pattern. See #20. + # be done on source branches with a certain pattern. See ACCESS-NRI/access-om2-configs#20. + if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && startsWith(github.base_ref, 'release-') && startsWith(github.head_ref, 'dev-') + needs: + - commit-check runs-on: ubuntu-latest permissions: pull-requests: write @@ -47,20 +67,6 @@ jobs: with: ref: ${{ github.head_ref }} - - name: Check Source - run: | - if [[ "${{ startsWith(github.head_ref, 'dev-') }}" == "false" ]]; then - echo "::error::Source branch ${{ github.head_ref }} doesn't match 'dev-*'" - exit 1 - fi - - - name: Check Target - run: | - if [[ "${{ startsWith(github.base_ref, 'release-') }}" == "false" ]]; then - echo "::error::Target branch ${{ github.base_ref }} doesn't match 'release-*'" - exit 1 - fi - - name: Compare Source and Target Config Names # In this step, we cut the 'dev-' and 'release-' to compare config names directly. run: | @@ -77,24 +83,89 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} BODY: | - :x: Automated testing cannot be run on this branch :x: + :x: Automated Reproducibility testing cannot be run on this branch :x: Source and Target branches must be of the form `dev-` and `release-` respectively, and `` must match between them. Rename the Source branch or check the Target branch, and try again. run: gh pr comment --body '${{ env.BODY }}' + qa-ci: + # Run quick, non-HPC tests on the runner. + name: QA CI Checks + needs: + - commit-check + if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME + runs-on: ubuntu-latest + permissions: + checks: write + steps: + - name: Checkout PR ${{ github.event.action.pull_request.number }} + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + path: pr + + - name: Checkout Tests + uses: actions/checkout@v4 + with: + ref: main + path: pytest + + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + + - name: Install requirements.txt + working-directory: ./pytest + run: pip install -r ./test/requirements.txt + + - name: Get Final List of Markers + id: pytest-markers + run: | + if [[ ${{ inputs.qa-pytest-add-model-markers }} == "true" ]]; then + echo "markers=${{ inputs.qa-pytest-markers }} or access_om2" >> $GITHUB_OUTPUT + else + echo "markers=${{ inputs.qa-pytest-markers }}" >> $GITHUB_OUTPUT + fi + + - name: Invoke Simple CI Pytests + # We continue on error because we will let the checks generated in + # the next step speak to the state of the testing + continue-on-error: true + working-directory: ./pr + run: | + echo "Running pytest using '-m ${{ steps.pytest-markers.outputs.markers }}'" + pytest ../pytest/test \ + -m '${{ steps.pytest-markers.outputs.markers }}' \ + --target-branch '${{ github.base_ref }}' \ + --junitxml=./test_report.xml + + - name: Parse Test Report + id: tests + uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 + with: + files: ./pr/test_report.xml + comment_mode: off + check_run: true + check_name: QA Test Results + compare_to_earlier_commit: false + report_individual_runs: true + report_suite_logs: any + repro-ci: - # run the given config on the deployment GitHub Environment (`environment-name`) and + # Run the given config on the deployment GitHub Environment (`environment-name`) and # upload the checksums and test details needs: - commit-check - branch-check - if: needs.commit-check.outputs.authorship != 'github-actions' && needs.branch-check.result == 'success' + if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && needs.branch-check.result == 'success' uses: access-nri/reproducibility/.github/workflows/checks.yml@main with: model-name: access-om2 environment-name: Gadi config-tag: ${{ github.head_ref }} - test-markers: checksum + test-markers: ${{ inputs.repro-pytest-markers }} secrets: inherit permissions: contents: write @@ -138,6 +209,7 @@ jobs: files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml comment_mode: off check_run: true + check_name: Repro Test Results compare_to_earlier_commit: false report_individual_runs: true report_suite_logs: any @@ -153,6 +225,38 @@ jobs: echo "result=pass" >> $GITHUB_OUTPUT fi + bump-check: + name: Version Bump Check + # Check that the `.version` in the metadata.yaml has been modified in + # this PR. + needs: + - repro-ci + runs-on: ubuntu-latest + steps: + - name: Checkout PR Target + uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + path: target + + - name: Checkout PR Source + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + path: source + + - name: Modification Check + run: | + target=$(yq e '.version' ./target/metadata.yaml) + source=$(yq e '.version' ./source/metadata.yaml) + + if [[ "${source}" != "${target}" && "${source}" != "null" ]]; then + echo "::notice::The version has been modified to ${target}. Merging is now availible" + else + echo "::error::The version has not been modified in this PR. Merging is disallowed until an appropriate '!bump' is issued" + exit 1 + fi + result: name: Repro Result Notifier # Notify the PR of the result of the Repro check @@ -171,35 +275,19 @@ jobs: ref: ${{ github.head_ref }} - name: Successful Release Comment - if: needs.check-checksum.outputs.result == 'pass' && startsWith(github.base_ref, 'release-') + if: needs.check-checksum.outputs.result == 'pass' env: BODY: | :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :white_check_mark: For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}. - Consider bumping the minor version of `access-om2-configs` - to bump the version, comment `!bump minor`. The meaning of these version bumps is explained in the README.md, under `Config Tags`. - run: gh pr comment --body '${{ env.BODY }}' - - - name: Successful Dev Comment - if: needs.check-checksum.outputs.result == 'pass' && startsWith(github.base_ref, 'dev-') - env: - BODY: | - :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Dev` config. :white_check_mark: - For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}. + You must bump the minor version of `access-om2-configs` - to bump the version, comment `!bump minor` or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`. run: gh pr comment --body '${{ env.BODY }}' - name: Failed Release Comment - if: needs.check-checksum.outputs.result == 'fail' && startsWith(github.base_ref, 'release-') + if: needs.check-checksum.outputs.result == 'fail' env: BODY: | :x: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :x: For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}. - You must bump the major version of `access-om2-configs` before this PR is merged to account for this - to bump the version, comment `!bump major`. The meaning of these version bumps is explained in the README.md, under `Config Tags`. - run: gh pr comment --body '${{ env.BODY }}' - - - name: Failed Dev Comment - if: needs.check-checksum.outputs.result == 'fail' && startsWith(github.base_ref, 'dev-') - env: - BODY: | - :warning: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Dev` config. :warning: - For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}. + You must bump the major version of `access-om2-configs` before this PR is merged to account for this - to bump the version, comment `!bump major`or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`. run: gh pr comment --body '${{ env.BODY }}' diff --git a/.github/workflows/pr-2-confirm.yml b/.github/workflows/pr-2-confirm.yml index adbcb472..c4987847 100644 --- a/.github/workflows/pr-2-confirm.yml +++ b/.github/workflows/pr-2-confirm.yml @@ -58,6 +58,12 @@ jobs: id: bump run: | version=$(yq '.version' metadata.yaml) + + if [[ "${version}" == "null" ]]; then + echo "before=null" >> $GITHUB_OUTPUT + echo "after=1.0" >> $GITHUB_OUTPUT + fi + regex="([0-9]+)\.([0-9]+)" if [[ $version =~ $regex ]]; then major_version="${BASH_REMATCH[1]}" @@ -118,8 +124,8 @@ jobs: - name: Commit and Push Updates run: | - git config user.name github-actions - git config user.email github-actions@github.com + git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} + git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} if [[ "${{ needs.bump-version.outputs.type }}" == "minor" ]]; then git commit -am "Bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}" diff --git a/.github/workflows/schedule-2-start.yml b/.github/workflows/schedule-2-start.yml index e0a9a1ee..f782ab39 100644 --- a/.github/workflows/schedule-2-start.yml +++ b/.github/workflows/schedule-2-start.yml @@ -80,7 +80,7 @@ jobs: MODEL_URL: https://github.com/ACCESS-NRI/access-om2 CONFIG: access-om2-configs CONFIG_URL: https://github.com/ACCESS-NRI/access-om2-configs - TAG_URL: https://github.com/ACCESS-NRI/access-om2-configs/tags/${{ needs.check-checksum.outputs.compared-checksum-version }} + TAG_URL: https://github.com/ACCESS-NRI/access-om2-configs/releases/tag/${{ needs.check-checksum.outputs.compared-checksum-version }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} steps: - name: Create issue diff --git a/README-DEV.md b/README-DEV.md index 4a6a0239..8f814645 100644 --- a/README-DEV.md +++ b/README-DEV.md @@ -10,9 +10,9 @@ The Reproducibility CI is comprised of two main triggers: on Pull Request and Sc ### Triggered On Pull Request -This pipeline uses the `pr-1-ci.yml` workflow for the majority of the PR lifecycle. +This pipeline uses the `pr-1-ci.yml` workflow for the majority of the PR lifecycle. The `main` version of this workflow is called by `call-pr-1-ci.yml` workflow on the config branch. -It also uses `pr-2-confirm.yml`, and `pr-3-bump-tag.yml` to control supplementary actions like bumping the `.version` field in the `metadata.yaml` file correctly and committing the checksums, and updating the config tag on merge. +It also uses `pr-2-confirm.yml`, and `pr-3-bump-tag.yml` (via `call-pr-3-bump-tag.yml` on the config branch) to control supplementary actions like bumping the `.version` field in the `metadata.yaml` file correctly and committing the checksums, and updating the config tag on merge. The overall pipeline looks like this: @@ -22,6 +22,13 @@ Merge: `pr-3-bump-tag.yml` #### The PR CI Lifecycle: `pr-1-ci.yml` This file does the bulk of the handling of the PR. +It contains the following inputs, when the PR-triggered `call-pr-1-ci.yml` calls it: + +| Name | Type | Description | Required | Default | Example | +| ---- | ---- | ----------- | -------- | ------- | ------- | +| `qa-pytest-markers` | `string` | Markers used for the pytest QA CI checks, in the python format | `false` | `config or metadata` | `config or metadata or highres` | +| `qa-pytest-add-model-markers` | `boolean` | Markers used for the pytest QA CI checks, in the python format | `false` | `false` | `true` | +| `repro-pytest-markers` | `string` | Markers used for the pytest repro CI checks, in the python format | `false` | `checksum` | `checksum or performance` | ##### `commit-check` @@ -29,6 +36,14 @@ The first job, `commit-check`, is used to short-circuit execution of this workfl `github-actions` authors commit to the `testing` directory and `metadata.yaml` files in the PR. +##### `branch-check` + +This job is used as a check before running [`repro-ci`](#repro-ci) checks, which are only run on PRs `dev-*` -> `release-*`. It also makes sure that the branches are formatted correctly. + +##### `qa-ci` + +These checks are runner-hosted, quick configuration sanity checks that will always fire on PRs into `release-*` or `dev-*`. + ##### `repro-ci` This job uses a reusable workflow that runs a given model on Gadi using a particular config tag (in this case, it'll be the source PR branch), and uploads the checksum as an artifact named `-`, for example, `access-om2-release-config-fix`. diff --git a/test/conftest.py b/test/conftest.py index 58d163de..d7f7e372 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,8 @@ import pytest from pathlib import Path +import yaml + @pytest.fixture(scope="session") def output_path(request): @@ -37,6 +39,32 @@ def checksum_path(request, control_path): return Path(path) +@pytest.fixture(scope="session") +def metadata(control_path: Path): + """Read the metadata file in the control directory""" + metadata_path = control_path / 'metadata.yaml' + with open(metadata_path) as f: + content = yaml.safe_load(f) + return content + + +@pytest.fixture(scope="session") +def config(control_path: Path): + """Read the config file in the control directory""" + config_path = control_path / 'config.yaml' + with open(config_path) as f: + config_content = yaml.safe_load(f) + return config_content + + +@pytest.fixture(scope="session") +def target_branch(request): + """Set the target branch - i.e., the branch the configuration will be + merged into. This used is to infer configuration information, if the + configuration branches follow a common naming scheme (e.g. ACCESS-OM2)""" + return request.config.getoption('--target-branch') + + # Set up command line options and default for directory paths def pytest_addoption(parser): """Attaches optional command line arguments""" @@ -51,15 +79,25 @@ def pytest_addoption(parser): parser.addoption("--checksum-path", action="store", help="Specify the checksum file to compare against") + + parser.addoption("--target-branch", + action="store", + help="Specify the target branch name") def pytest_configure(config): config.addinivalue_line( "markers", "slow: mark tests as slow (deselect with '-m \"not slow\"')" ) + config.addinivalue_line( + "markers", "test: mark tests as testing test functionality" + ) config.addinivalue_line( "markers", "checksum: mark tests to run as part of reproducibility CI tests" ) config.addinivalue_line( - "markers", "test: mark tests as testing test functionality" + "markers", "config: mark as configuration tests in quick QA CI checks" + ) + config.addinivalue_line( + "markers", "access_om2: mark as access-om2 specific tests in quick QA CI checks" ) diff --git a/test/requirements.txt b/test/requirements.txt index 967a4754..9e995937 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,5 @@ pytest==8.0.1 jsonschema==4.21.1 +requests +PyYAML +f90nml>=0.16 diff --git a/test/test_access_om2_config.py b/test/test_access_om2_config.py new file mode 100644 index 00000000..dcd567d7 --- /dev/null +++ b/test/test_access_om2_config.py @@ -0,0 +1,132 @@ +import re + +import pytest +import f90nml +import warnings + +from util import get_git_branch_name + +# Mutually exclusive topic keywords +TOPIC_KEYWORDS = { + 'spatial extent': {'global', 'regional'}, + 'forcing product': {'JRA55', 'ERA5'}, + 'forcing mode': {'repeat-year', 'ryf', 'repeat-decade', 'rdf', + 'interannual', 'iaf'}, + 'model': {'access-om2', 'access-om2-025', 'access-om2-01'} +} + + +class AccessOM2Branch: + """Use the naming patterns of the branch name to infer informatiom of + the ACCESS-OM2 config""" + + def __init__(self, branch_name): + self.branch_name = branch_name + self.set_resolution() + + self.is_high_resolution = self.resolution in ['025deg', '01deg'] + self.is_bgc = 'bgc' in branch_name + + def set_resolution(self): + # Resolutions are ordered, so the start of the list are matched first + resolutions = ['025deg', '01deg', '1deg'] + self.resolution = None + for res in resolutions: + if res in self.branch_name: + self.resolution = res + return + + pytest.fail( + f"Branch name {self.branch_name} has an unknown resolution. " + + f"Current supported resolutions: {', '.join(resolutions)}" + ) + + +@pytest.fixture(scope="class") +def branch(control_path, target_branch): + branch_name = target_branch + if branch_name is None: + # Default to current branch name + branch_name = get_git_branch_name(control_path) + assert branch_name is not None, ( + f"Failed getting git branch name of control path: {control_path}" + ) + warnings.warn( + "Target branch is not specifed, defaulting to current git branch: " + f"{branch_name}. As some ACCESS-OM2 tests infer information, " + "such as resolution, from the target branch name, some tests may " + "not be run. To set use --target-branch flag in pytest call" + ) + + return AccessOM2Branch(branch_name) + + +@pytest.mark.access_om2 +class TestAccessOM2: + """ACCESS-OM2 Specific configuration and metadata tests""" + + def test_mppncombine_fast_collate_exe(self, config, branch): + if branch.is_high_resolution: + pattern = r'/g/data/vk83/apps/mppnccombine-fast/.*/bin/mppnccombine-fast' + if 'collate' in config: + assert re.match(pattern, config['collate']['exe']), ( + "Expect collate executable set to mppnccombine-fast" + ) + + def test_metadata_realm(self, metadata, branch): + expected_realms = {'ocean', 'seaIce'} + expected_config = 'realm:\n - ocean\n - seaIce' + if branch.is_bgc: + expected_realms.add('ocnBgchem') + expected_config += '\n - ocnBgchem' + + assert ('realm' in metadata + and set(metadata['realm']) == expected_realms), ( + 'Expected metadata realm set to:\n' + expected_config + ) + + def test_restart_period(self, branch, control_path): + accessom2_nml_path = control_path / 'accessom2.nml' + assert accessom2_nml_path.exists() + + accessom2_nml = f90nml.read(accessom2_nml_path) + restart_period = accessom2_nml['date_manager_nml']['restart_period'] + + if branch.resolution == '1deg': + expected_period = [5, 0, 0] + elif branch.resolution == '025deg': + if branch.is_bgc: + expected_period = [1, 0, 0] + else: + expected_period = [2, 0, 0] + elif branch.resolution == '01deg': + if branch.is_bgc: + expected_period = [0, 1, 0] + else: + expected_period = [0, 3, 0] + else: + pytest.fail( + f"The expected restart period is not defined for given " + f"resolution: {branch.resolution}" + ) + assert restart_period == expected_period + + def test_metadata_keywords(self, metadata): + + assert 'keywords' in metadata + metadata_keywords = set(metadata['keywords']) + + expected_keywords = set() + for topic, keywords in TOPIC_KEYWORDS.items(): + mutually_exclusive = metadata_keywords.intersection(keywords) + assert len(mutually_exclusive) <= 1, ( + f"Topic {topic} has multiple mutually exlusive keywords: " + + str(mutually_exclusive) + ) + + expected_keywords = expected_keywords.union(keywords) + + unrecognised_keywords = metadata_keywords.difference(expected_keywords) + assert len(unrecognised_keywords) == 0, ( + f"Metadata has unrecognised keywords: {unrecognised_keywords}" + ) diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 00000000..2dca7ac9 --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,181 @@ +"""Tests for checking configs and valid metadata files""" + +from pathlib import Path +import re +import warnings + +import pytest +import requests +import jsonschema +import yaml + +BASE_SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema" +BASE_SCHEMA_PATH = "au.org.access-nri/model/output/experiment-metadata" +DEFAULT_SCHEMA_VERSION = "1-0-0" +DEFAULT_SCHEMA_COMMIT = "bfc15e3c6fa20d492ccfa0c4706805d64c531e7c" + + +@pytest.fixture(scope="class") +def exe_manifest_fullpaths(control_path: Path): + manifest_path = control_path / 'manifests' / 'exe.yaml' + with open(manifest_path) as f: + _, data = yaml.safe_load_all(f) + exe_fullpaths = {item['fullpath'] for item in data.values()} + return exe_fullpaths + + +def insist_array(str_or_array): + if isinstance(str_or_array, str): + str_or_array = [str_or_array,] + return str_or_array + + +@pytest.mark.config +class TestConfig: + """General configuration tests""" + + @pytest.mark.parametrize( + "field", ["project", "shortpath"] + ) + def test_field_is_not_defined(self, config, field): + assert field not in config, ( + f"{field} should not be defined: '{field}: {config[field]}'" + ) + + def test_absolute_input_paths(self, config): + for path in insist_array(config.get('input', [])): + assert Path(path).is_absolute(), ( + f"Input path should be absolute: {path}" + ) + + def test_absolute_submodel_input_paths(self, config): + for model in config.get('submodels', []): + for path in insist_array(model.get('input', [])): + assert Path(path).is_absolute(), ( + f"Input path for {model['name']} submodel should be " + + f" absolute: {path}" + ) + + def test_no_storage_qsub_flags(self, config): + qsub_flags = config.get('qsub_flags', '') + assert 'storage' not in qsub_flags, ( + "Storage flags defined in qsub_flags will be silently ignored" + ) + + def test_runlog_is_on(self, config): + runlog_config = config.get('runlog', {}) + if isinstance(runlog_config, bool): + runlog_enabled = runlog_config + else: + runlog_enabled = runlog_config.get('enable', True) + assert runlog_enabled + + def test_absolute_exe_path(self, config): + assert 'exe' not in config or Path(config['exe']).is_absolute(), ( + f"Executable for model should be an absolute path: {config['exe']}" + ) + + def test_absolute_submodel_exe_path(self, config): + for model in config.get('submodels', []): + if 'exe' not in model: + # Allow models such as couplers that have no executable + if 'ncpus' in model and model['ncpus'] != 0: + pytest.fail(f"No executable for submodel {model['name']}") + continue + + assert Path(model['exe']).is_absolute(), ( + f"Executable for {model['name']} submodel should be " + + f"an absolute path: {config['exe']}" + ) + + def test_exe_paths_in_manifest(self, config, exe_manifest_fullpaths): + if 'exe' in config: + assert config['exe'] in exe_manifest_fullpaths, ( + f"Model executable path should be in Manifest file " + + f"(e.g. manifests/exe.yaml): {config['exe']}" + ) + + def test_sub_model_exe_paths_in_manifest(self, config, + exe_manifest_fullpaths): + for model in config.get('submodels', []): + if 'exe' in model: + assert model['exe'] in exe_manifest_fullpaths, ( + f"Submodel {model['name']} executable path should be in " + + f"Manifest file (e.g. manifests/exe.yaml): {config['exe']}" + ) + + def test_restart_freq_is_date_based(self, config): + assert "restart_freq" in config, "Restart frequency should be defined" + frequency = config["restart_freq"] + # String of an integer followed by a YS/MS/W/D/H/T/S unit, + # e.g. 1YS for 1 year-start + pattern = r'^\d+(YS|MS|W|D|H|T|S)$' + assert isinstance(frequency, str) and re.match(pattern, frequency), ( + "Restart frequency should be date-based: " + + f"'restart_freq: {frequency}'" + ) + + def test_sync_is_not_enabled(self, config): + if 'sync' in config and 'enable' in config['sync']: + assert not config['sync']['enable'], ( + "Sync to remote archive should not be enabled" + ) + + def test_sync_path_is_not_set(self, config): + if 'sync' in config: + assert not ('path' in config['sync'] + and config['sync']['path'] is not None), ( + "Sync path to remote archive should not be set" + ) + + def test_manifest_reproduce_exe_is_on(self, config): + manifest_reproduce = config.get('manifest', {}).get('reproduce', {}) + assert 'exe' in manifest_reproduce and manifest_reproduce['exe'], ( + "Executable reproducibility should be enforced, e.g set:\n" + + "manifest:\n reproduce:\n exe: True" + ) + + def test_no_scripts_in_top_level_directory(self, control_path): + exts = {".py", ".sh"} + scripts = [p for p in control_path.iterdir() if p.suffix in exts] + assert scripts == [], ( + "Scripts in top-level directory should be moved to a " + + "'tools' sub-directory" + ) + + def test_validate_metadata(self, metadata): + # Get schema from Github + schema_version = metadata.get("schema_version", DEFAULT_SCHEMA_VERSION) + schema_path = f"{BASE_SCHEMA_PATH}/{schema_version}.json" + + url = f"{BASE_SCHEMA_URL}/main/{schema_path}" + response = requests.get(url) + if response.status_code != 200: + # Use default schema + warnings.warn( + f"Failed to retrieve schema from url: {url}\n" + + f"Defaulting to schema version: {DEFAULT_SCHEMA_VERSION}" + ) + schema_path = f"{BASE_SCHEMA_PATH}/{DEFAULT_SCHEMA_VERSION}.json" + + url = f"{BASE_SCHEMA_URL}/{DEFAULT_SCHEMA_COMMIT}/{schema_path}" + response = requests.get(url) + assert response.status_code == 200 + schema = response.json() + + # In schema version (1-0-0), required fields are name, experiment_uuid, + # description and long_description. As name & experiment_uuid are + # generated for running experiments, the required fields are removed + # from the schema validation for now + schema.pop('required') + + # Validate field names and types + jsonschema.validate(instance=metadata, schema=schema) + + @pytest.mark.parametrize( + "field", + ["description", "notes", "keywords", "nominal resolution", "version", + "reference", "license", "url", "model", "realm"] + ) + def test_metadata_contains_fields(self, field, metadata): + assert field in metadata, f"{field} field shoud be defined in metadata" diff --git a/test/util.py b/test/util.py index bfb84042..d3212be0 100644 --- a/test/util.py +++ b/test/util.py @@ -18,3 +18,16 @@ def wait_for_qsub(run_id): if 'Job has finished' in qsub_out: break + + +def get_git_branch_name(path): + """Get the git branch name of the given git directory""" + try: + cmd = 'git rev-parse --abbrev-ref HEAD' + result = sp.check_output(cmd, shell=True, + cwd=path).strip() + # Decode byte string to string + branch_name = result.decode('utf-8') + return branch_name + except sp.CalledProcessError: + return None