fix: unit test execution in PyCharm (oscal-compass#1755)

* Where does pathlib.Path('tests/data') resolve the path from? It's relative to the current working directory. * Where is the tests/data directory? Relative to the tests. * PyCharm has quirks related to the current working directory when running and debugging tests. * By using __file__ to resolve the test data directory, we remove the non-guaranteed assumption that the current working directory is always at the root of the repository, and we replace that assumption with a value that is always correct regardless of current working directory. * As a result, the unit tests can now be debugged in PyCharm, unit tests can be executed from any directory, and test execution continues to work in all the places it originally worked. Some of the handling is slightly messy but that can be cleaned up by making the task config files themselves not dependent on the CWD Signed-off-by: d10n <[email protected]>
d10n · Nov 19, 2024 · b91bb7d · b91bb7d
1 parent 0bd1b70
commit b91bb7d
Show file tree

Hide file tree

Showing 23 changed files with 496 additions and 195 deletions.
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -259,7 +259,9 @@ def tmp_empty_cwd(tmp_path: pathlib.Path) -> Iterator[pathlib.Path]:
 @pytest.fixture(scope='function')
 def testdata_dir() -> pathlib.Path:
     """Return absolute path to test data directory."""
-    test_data_source = pathlib.Path('tests/data')
+    test_dir = pathlib.Path(__file__).parent.resolve()
+    data_path = test_dir / 'data'
+    test_data_source = pathlib.Path(data_path)
     return test_data_source.resolve()
 
 

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -17,6 +17,7 @@
 
 import argparse
 import difflib
+import functools
 import logging
 import os
 import pathlib
@@ -53,31 +54,54 @@
 
 logger = logging.getLogger(__name__)
 
-BASE_TMP_DIR = pathlib.Path('tests/__tmp_path').resolve()
-YAML_TEST_DATA_PATH = pathlib.Path('tests/data/yaml/').resolve()
-JSON_TEST_DATA_PATH = pathlib.Path('tests/data/json/').resolve()
-ENV_TEST_DATA_PATH = pathlib.Path('tests/data/env/').resolve()
-JSON_NIST_DATA_PATH = pathlib.Path('nist-content/nist.gov/SP800-53/rev5/json/').resolve()
+TEST_DIR = pathlib.Path(__file__).parent.resolve()
+BASE_TMP_DIR = pathlib.Path(TEST_DIR / '__tmp_path').resolve()
+YAML_TEST_DATA_PATH = pathlib.Path(TEST_DIR / 'data/yaml/').resolve()
+JSON_TEST_DATA_PATH = pathlib.Path(TEST_DIR / 'data/json/').resolve()
+ENV_TEST_DATA_PATH = pathlib.Path(TEST_DIR / 'data/env/').resolve()
+JSON_NIST_DATA_PATH = pathlib.Path(TEST_DIR / '../nist-content/nist.gov/SP800-53/rev5/json/').resolve()
 JSON_NIST_CATALOG_NAME = 'NIST_SP-800-53_rev5_catalog.json'
 JSON_NIST_PROFILE_NAME = 'NIST_SP-800-53_rev5_MODERATE-baseline_profile.json'
-JSON_NIST_REV_4_DATA_PATH = pathlib.Path('nist-content/nist.gov/SP800-53/rev4/json/').resolve()
+JSON_NIST_REV_4_DATA_PATH = pathlib.Path(TEST_DIR / '../nist-content/nist.gov/SP800-53/rev4/json/').resolve()
 JSON_NIST_REV_4_CATALOG_NAME = 'NIST_SP-800-53_rev4_catalog.json'
 JSON_NIST_REV_5_CATALOG_NAME = 'nist-rev5-catalog-full.json'
 JSON_NIST_REV_4_PROFILE_NAME = 'NIST_SP-800-53_rev4_MODERATE-baseline_profile.json'
 SIMPLIFIED_NIST_CATALOG_NAME = 'simplified_nist_catalog.json'
 SIMPLIFIED_NIST_PROFILE_NAME = 'simplified_nist_profile.json'
-TASK_XLSX_OUTPUT_PATH = pathlib.Path('tests/data/tasks/xlsx/output').resolve()
+TASK_XLSX_OUTPUT_PATH = pathlib.Path(TEST_DIR / 'data/tasks/xlsx/output').resolve()
 
 CATALOGS_DIR = 'catalogs'
 PROFILES_DIR = 'profiles'
 COMPONENT_DEF_DIR = 'component-definitions'
 
-NIST_EXAMPLES = pathlib.Path('nist-content/examples')
+NIST_EXAMPLES = pathlib.Path(TEST_DIR / '../nist-content/examples')
 NIST_SAMPLE_CD_JSON = NIST_EXAMPLES / 'component-definition' / 'json' / 'example-component.json'
 
 NEW_MODEL_AGE_SECONDS = 100
 
 
+def set_cwd_unsafe(cwd: pathlib.Path = TEST_DIR):
+    """Set the current working directory to the test directory.
+
+    Not safe for concurrent tests which may change directory.
+    """
+
+    def decorator(f):
+
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            original_cwd = os.getcwd()
+            os.chdir(cwd)
+            try:
+                f(*args, **kwargs)
+            finally:
+                os.chdir(original_cwd)
+
+        return wrapper
+
+    return decorator
+
+
 def clean_tmp_path(tmp_path: pathlib.Path):
     """Clean tmp directory."""
     if tmp_path.exists():

diff --git a/tests/trestle/core/commands/validate_test.py b/tests/trestle/core/commands/validate_test.py
@@ -46,8 +46,6 @@
 from trestle.oscal.common import ResponsibleParty, Role
 from trestle.oscal.component import ComponentDefinition, ControlImplementation
 
-test_data_dir = pathlib.Path('tests/data').resolve()
-
 md_path = 'md_comp'
 
 
@@ -58,16 +56,18 @@
         ('my_test_model', '-t', False), ('my_test_model', '-a', False), ('my_test_model', '-x', False)
     ]
 )
-def test_validation_happy(name, mode, parent, tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+def test_validation_happy(
+    name, mode, parent, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch
+) -> None:
     """Test successful validation runs."""
     (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True)
     (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2').mkdir(exist_ok=True, parents=True)
     shutil.copyfile(
-        test_data_dir / 'json/minimal_catalog.json',
+        testdata_dir / 'json/minimal_catalog.json',
         tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model/catalog.json'
     )
     shutil.copyfile(
-        test_data_dir / 'json/minimal_catalog.json',
+        testdata_dir / 'json/minimal_catalog.json',
         tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2/catalog.json'
     )
 
@@ -101,17 +101,17 @@ def test_validation_happy(name, mode, parent, tmp_trestle_dir: pathlib.Path, mon
     ]
 )
 def test_validation_unhappy(
-    name, mode, parent, status, tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch
+    name, mode, parent, status, tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch
 ) -> None:
     """Test failure modes of validation."""
     (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model').mkdir(exist_ok=True, parents=True)
     (tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2').mkdir(exist_ok=True, parents=True)
     shutil.copyfile(
-        test_data_dir / 'json/minimal_catalog_bad_oscal_version.json',
+        testdata_dir / 'json/minimal_catalog_bad_oscal_version.json',
         tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model/catalog.json'
     )
     shutil.copyfile(
-        test_data_dir / 'json/minimal_catalog.json',
+        testdata_dir / 'json/minimal_catalog.json',
         tmp_trestle_dir / test_utils.CATALOGS_DIR / 'my_test_model2/catalog.json'
     )
 
@@ -421,11 +421,13 @@ def test_period(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None
         pass
 
 
-def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+def test_validate_component_definition(
+    tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch
+) -> None:
     """Test validation of Component Definition."""
     jfile = 'component-definition.json'
 
-    sdir = test_data_dir / 'validate' / 'component-definitions' / 'x1'
+    sdir = testdata_dir / 'validate' / 'component-definitions' / 'x1'
     spth = sdir / f'{jfile}'
 
     tdir = tmp_trestle_dir / test_utils.COMPONENT_DEF_DIR / 'my_test_model'
@@ -438,11 +440,13 @@ def test_validate_component_definition(tmp_trestle_dir: pathlib.Path, monkeypatc
     test_utils.execute_command_and_assert(validate_command, 0, monkeypatch)
 
 
-def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+def test_validate_component_definition_ports(
+    tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch
+) -> None:
     """Test validation of ports in Component Definition."""
     jfile = 'component-definition.json'
 
-    sdir = test_data_dir / 'validate' / 'component-definitions' / 'x2'
+    sdir = testdata_dir / 'validate' / 'component-definitions' / 'x2'
     spth = sdir / f'{jfile}'
 
     tdir = tmp_trestle_dir / test_utils.COMPONENT_DEF_DIR / 'my_test_model'
@@ -455,11 +459,13 @@ def test_validate_component_definition_ports(tmp_trestle_dir: pathlib.Path, monk
     test_utils.execute_command_and_assert(validate_command, 0, monkeypatch)
 
 
-def test_validate_component_definition_ports_invalid(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+def test_validate_component_definition_ports_invalid(
+    tmp_trestle_dir: pathlib.Path, testdata_dir: pathlib.Path, monkeypatch: MonkeyPatch
+) -> None:
     """Test validation of ports in Component Definition."""
     jfile = 'component-definition.json'
 
-    sdir = test_data_dir / 'validate' / 'component-definitions' / 'x3'
+    sdir = testdata_dir / 'validate' / 'component-definitions' / 'x3'
     spth = sdir / f'{jfile}'
 
     tdir = tmp_trestle_dir / test_utils.COMPONENT_DEF_DIR / 'my_test_model'

diff --git a/tests/trestle/core/control_io_test.py b/tests/trestle/core/control_io_test.py
@@ -338,10 +338,12 @@ def test_get_control_param_dict(tmp_trestle_dir: pathlib.Path) -> None:
 
 
 @pytest.mark.parametrize('overwrite_header_values', [True, False])
-def test_write_control_header_params(overwrite_header_values, tmp_path: pathlib.Path) -> None:
+def test_write_control_header_params(
+    overwrite_header_values, tmp_path: pathlib.Path, testdata_dir: pathlib.Path
+) -> None:
     """Test write/read of control header params."""
     # orig file just has one param ac-1_prm_3
-    src_control_path = pathlib.Path('tests/data/author/controls/control_with_components_and_params.md')
+    src_control_path = pathlib.Path(testdata_dir / 'author/controls/control_with_components_and_params.md')
     # header has two params - 3 and 4
     header = {
         const.SET_PARAMS_TAG: {

diff --git a/tests/trestle/core/draw_io_test.py b/tests/trestle/core/draw_io_test.py
@@ -21,6 +21,8 @@
 
 import pytest
 
+from tests.test_utils import TEST_DIR
+
 from trestle.common.err import TrestleError
 from trestle.core.draw_io import DrawIO, DrawIOMetadataValidator
 from trestle.core.markdown.markdown_validator import MarkdownValidator
@@ -42,20 +44,13 @@ def test_missing_file(tmp_path) -> None:
 @pytest.mark.parametrize(
     'file_path, metadata_exists, metadata_valid',
     [
-        (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_bad_metadata_extra_fields_compressed.drawio'),
-            True,
-            False
-        ),
-        (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_bad_metadata_missing_fields_compressed.drawio'),
-            True,
-            False
-        ), (pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'), True, True),
-        (pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_no_metadata_compressed.drawio'), False, False),
-        (pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_no_metadata_uncompressed.drawio'), False, False),
-        (pathlib.Path('tests/data/author/0.0.1/drawio/two_tabs_metadata_compressed.drawio'), True, True),
-        (pathlib.Path('tests/data/author/0.0.1/drawio/two_tabs_metadata_second_tab_compressed.drawio'), True, False)
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_bad_metadata_extra_fields_compressed.drawio', True, False),
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_bad_metadata_missing_fields_compressed.drawio', True, False),
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio', True, True),
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_no_metadata_compressed.drawio', False, False),
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_no_metadata_uncompressed.drawio', False, False),
+        (TEST_DIR / 'data/author/0.0.1/drawio/two_tabs_metadata_compressed.drawio', True, True),
+        (TEST_DIR / 'data/author/0.0.1/drawio/two_tabs_metadata_second_tab_compressed.drawio', True, False)
     ]
 )
 def test_valid_drawio(file_path: pathlib.Path, metadata_exists: bool, metadata_valid: bool) -> None:
@@ -78,9 +73,9 @@ def test_valid_drawio(file_path: pathlib.Path, metadata_exists: bool, metadata_v
 @pytest.mark.parametrize(
     'bad_file_name',
     [
-        (pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_no_metadata_uncompressed_mangled.drawio')),
-        (pathlib.Path('tests/data/author/0.0.1/drawio/not_mxfile.drawio')),
-        (pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_no_metadata_bad_internal_structure.drawio'))
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_no_metadata_uncompressed_mangled.drawio'),
+        (TEST_DIR / 'data/author/0.0.1/drawio/not_mxfile.drawio'),
+        (TEST_DIR / 'data/author/0.0.1/drawio/single_tab_no_metadata_bad_internal_structure.drawio')
     ]
 )
 def test_bad_drawio_files(bad_file_name: pathlib.Path) -> None:
@@ -93,32 +88,32 @@ def test_bad_drawio_files(bad_file_name: pathlib.Path) -> None:
     'template_file, sample_file, must_be_first_tab, metadata_valid',
     [
         (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'),
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'),
+            TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio',
+            TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio',
             True,
             True
         ),
         (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'),
-            pathlib.Path('tests/data/author/0.0.1/drawio/two_tabs_metadata_compressed.drawio'),
+            TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio',
+            TEST_DIR / 'data/author/0.0.1/drawio/two_tabs_metadata_compressed.drawio',
             True,
             True
         ),
         (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'),
-            pathlib.Path('tests/data/author/0.0.1/drawio/two_tabs_metadata_second_tab_compressed.drawio'),
+            TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio',
+            TEST_DIR / 'data/author/0.0.1/drawio/two_tabs_metadata_second_tab_compressed.drawio',
             True,
             False
         ),
         (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'),
-            pathlib.Path('tests/data/author/0.0.1/drawio/two_tabs_metadata_second_tab_compressed.drawio'),
+            TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio',
+            TEST_DIR / 'data/author/0.0.1/drawio/two_tabs_metadata_second_tab_compressed.drawio',
             False,
             True
         ),
         (
-            pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'),
-            pathlib.Path('tests/data/author/0.0.1/drawio/two_tabs_metadata_second_tab_bad_md.drawio'),
+            TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio',
+            TEST_DIR / 'data/author/0.0.1/drawio/two_tabs_metadata_second_tab_bad_md.drawio',
             False,
             False
         )
@@ -135,7 +130,7 @@ def test_valid_drawio_second_tab(
 
 def test_restructure_metadata():
     """Test Restructuring metadata."""
-    drawio_file = pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio')
+    drawio_file = TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'
     comparison_metadata = {'test': 'value', 'nested': {'test': 'value', 'extra': 'value', 'nested': {'test': 'value'}}}
     draw_io = DrawIO(drawio_file)
     metadata_flat = draw_io.get_metadata()[0]
@@ -146,9 +141,7 @@ def test_restructure_metadata():
 def test_write_metadata_compressed(tmp_path):
     """Test writing modified metadata to drawio file."""
     tmp_drawio_file = tmp_path / 'test.drawio'
-    shutil.copyfile(
-        pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio'), tmp_drawio_file
-    )
+    shutil.copyfile(TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_compressed.drawio', tmp_drawio_file)
     draw_io = DrawIO(tmp_drawio_file)
 
     diagram_idx = 0
@@ -171,9 +164,7 @@ def test_write_metadata_compressed(tmp_path):
 def test_write_metadata_uncompressed(tmp_path):
     """Test writing modified metadata to drawio file."""
     tmp_drawio_file = tmp_path / 'test.drawio'
-    shutil.copyfile(
-        pathlib.Path('tests/data/author/0.0.1/drawio/single_tab_metadata_uncompressed.drawio'), tmp_drawio_file
-    )
+    shutil.copyfile(TEST_DIR / 'data/author/0.0.1/drawio/single_tab_metadata_uncompressed.drawio', tmp_drawio_file)
     draw_io = DrawIO(tmp_drawio_file)
 
     diagram_idx = 0

diff --git a/tests/trestle/core/markdown/markdown_node_test.py b/tests/trestle/core/markdown/markdown_node_test.py
@@ -20,12 +20,14 @@
 
 import pytest
 
+from tests.test_utils import TEST_DIR
+
 import trestle.common.const as const
 from trestle.core.markdown.docs_markdown_node import DocsMarkdownNode, DocsSectionContent
 from trestle.core.markdown.markdown_api import MarkdownAPI
 
 
-@pytest.mark.parametrize('md_path', [(pathlib.Path('tests/data/markdown/valid_complex_md.md'))])
+@pytest.mark.parametrize('md_path', [(TEST_DIR / 'data/markdown/valid_complex_md.md')])
 def test_tree_text_equal_to_md(md_path: pathlib.Path) -> None:
     """Test tree construction."""
     contents = frontmatter.loads(md_path.open('r', encoding=const.FILE_ENCODING).read())
@@ -36,7 +38,7 @@ def test_tree_text_equal_to_md(md_path: pathlib.Path) -> None:
     assert markdown_wo_header == tree.content.raw_text
 
 
-@pytest.mark.parametrize('md_path', [(pathlib.Path('tests/data/markdown/valid_complex_md.md'))])
+@pytest.mark.parametrize('md_path', [(TEST_DIR / 'data/markdown/valid_complex_md.md')])
 def test_md_get_node_for_key(md_path: pathlib.Path) -> None:
     """Test node fetching."""
     contents = frontmatter.loads(md_path.open('r', encoding=const.FILE_ENCODING).read())
@@ -64,7 +66,7 @@ def test_md_get_node_for_key(md_path: pathlib.Path) -> None:
     assert node.key == '### 5.1.1 A deeper section 1'
 
 
-@pytest.mark.parametrize('md_path', [(pathlib.Path('tests/data/markdown/valid_complex_md.md'))])
+@pytest.mark.parametrize('md_path', [(TEST_DIR / 'data/markdown/valid_complex_md.md')])
 def test_md_content_is_correct(md_path: pathlib.Path) -> None:
     """Test that read content is correct."""
     contents = frontmatter.loads(md_path.open('r', encoding=const.FILE_ENCODING).read())
@@ -83,7 +85,7 @@ def test_md_content_is_correct(md_path: pathlib.Path) -> None:
     assert deep_node.content.text[1] == 'some very deep text'
 
 
-@pytest.mark.parametrize('md_path', [(pathlib.Path('tests/data/markdown/valid_complex_md.md'))])
+@pytest.mark.parametrize('md_path', [(TEST_DIR / 'data/markdown/valid_complex_md.md')])
 def test_md_headers_in_html_blocks_are_ignored(md_path: pathlib.Path) -> None:
     """Test that headers in the various html blocks are ignored."""
     contents = frontmatter.loads(md_path.open('r', encoding=const.FILE_ENCODING).read())