refactor tests with github actions

.github/workflows/run-tests.yml

@@ -47,17 +47,79 @@ jobs:
       - name: Type checking
         run: uv run mypy src tests
 
-  tests:
-    name: python-${{ matrix.python-version }} on ${{ matrix.os }}
+  tests_fast:
+    name: (fast) python-${{ matrix.python-version }} on ${{ matrix.os }}
 
     runs-on: ${{ matrix.os }}
 
     strategy:
       matrix:
         os:
           - ubuntu-latest
-          # - windows-latest
-          # - macos-latest
+          - windows-latest
+          - macos-latest
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
+
+      fail-fast: false
+
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache folder for traffic
+        uses: actions/cache@v4
+        id: cache-folder
+        with:
+          path: |
+            ~/.cache/traffic/
+          key: traffic-${{ hashFiles('uv.lock') }}
+
+      - name: Ubuntu system dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y libgdal-dev libgeos-dev libproj-dev proj-bin proj-data
+
+      - name: Install project
+        run: |
+          uv sync --dev --all-extras
+
+      - name: Run tests
+        env:
+          LD_LIBRARY_PATH: /usr/local/lib
+          TRAFFIC_NM_PATH: ""
+        run: |
+          uv run traffic cache --fill  # download files to have in cache first
+          uv run pytest -m "slow and extra"
+
+  tests_full:
+    name: (full) python-${{ matrix.python-version }} on ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
+
+    if: ${{ github.event_name != 'pull_request_target' }}
+
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          #- windows-latest
+          #- macos-latest
         python-version:
           - "3.10"
           - "3.11"
@@ -119,8 +181,6 @@ jobs:
         with:
           path: |
             ~/.cache/traffic/
-            ~/.cache/opensky/
-            ~/.cache/cartes/
           key: traffic-${{ hashFiles('uv.lock') }}
 
       - name: Ubuntu system dependencies
@@ -133,24 +193,12 @@ jobs:
           uv sync --dev --all-extras
 
       - name: Run tests
-        if: ${{ matrix.python-version == '3.12' || steps.cache-folder.outputs.cache-hit == 'true' }}
         env:
           LD_LIBRARY_PATH: /usr/local/lib
-          TRAFFIC_NOPLUGIN: ""
-          OPENSKY_USERNAME: ${{ secrets.OPENSKY_USERNAME }}
-          OPENSKY_PASSWORD: ${{ secrets.OPENSKY_PASSWORD }}
+          TRAFFIC_NM_PATH: /home/runner/work/traffic/traffic/traffic_data/airac_2111
         run: |
-          export TRAFFIC_CONFIG=$(uv run python -c "from traffic import config_file; print(config_file)")
-
-          sed -i "/nm_path =/ s,= $,= $PWD/traffic_data/airac_2111," $TRAFFIC_CONFIG
-          # sed -i "/pkcs12_filename =/ s,= .*$,= $PWD/traffic_data/CC0000007011_501_openssl.p12," $TRAFFIC_CONFIG
-          # sed -i "/pkcs12_password =/ s,= $,= $PKCS12_PASSWORD," $TRAFFIC_CONFIG
-
-          # Don't purge OpenSky cache files here :)
-          sed -i "/purge = / d" $TRAFFIC_CONFIG
-
           uv run traffic cache --fill  # download files to have in cache first
-          uv run pytest --cov --cov-report xml
+          uv run pytest --cov --cov-report xml -m ""
 
       - name: Upload coverage to Codecov
         if: ${{ github.event_name != 'pull_request_target' }}

pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
   "py7zr>=0.22.0",
   "pyarrow>=18.0.0",
   "pyopensky>=2.11",
+  "python-dotenv>=1.0.1",
   "rich>=13.9.4",
   "rs1090>=0.3.8",
   "typing-extensions>=4.12.2",
@@ -126,12 +127,17 @@ exclude = ["docs/_build"]
 
 
 [tool.pytest.ini_options]
-addopts = "--log-level=INFO --color=yes --doctest-modules --doctest-report ndiff"
+# add option --durations=10 to get an idea of slow tests
+addopts = "--log-level=INFO --color=yes --doctest-modules --doctest-report ndiff -m 'not slow and not extra'"
+doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS", "NUMBER"]
+markers = [
+  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+  "extra: marks tests as requiring private data (deselect with '-m \"not extra\"')",
+]
 testpaths = [
   "src/traffic/core/intervals.py",
   "src/traffic/core/time.py",
   "src/traffic/data/basic/",
   "src/traffic/data/datasets/",
   "tests",
 ]
-doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS", "NUMBER"]

src/traffic/__init__.py

@@ -1,25 +1,31 @@
 import configparser
 import logging
 import os
-from importlib.metadata import EntryPoint, entry_points, version
+from importlib.metadata import version
 from pathlib import Path
-from typing import Iterable
+from typing import TypedDict
 
+import dotenv
 from appdirs import user_cache_dir, user_config_dir
 
 import pandas as pd
 
 from . import visualize  # noqa: F401
 
 __version__ = version("traffic")
-__all__ = ["cache_dir", "config_dir", "config_file", "tqdm_style"]
+__all__ = ["cache_path", "config_dir", "config_file", "tqdm_style"]
 
 # Set up the library root logger
 _log = logging.getLogger(__name__)
 
+dotenv.load_dotenv()
+
 # -- Configuration management --
 
-config_dir = Path(user_config_dir("traffic"))
+if (xdg_config := os.environ.get("XDG_CONFIG_HOME")) is not None:
+    config_dir = Path(xdg_config) / "traffic"
+else:
+    config_dir = Path(user_config_dir("traffic"))
 config_file = config_dir / "traffic.conf"
 
 if not config_dir.exists():  # coverage: ignore
@@ -30,68 +36,94 @@
 config = configparser.ConfigParser()
 config.read(config_file.as_posix())
 
-# Check the config file for a cache directory. If not present
-# then use the system default cache path
-
-cache_dir = Path(user_cache_dir("traffic"))
-
-cache_dir_cfg = config.get("cache", "path", fallback="").strip()
-if cache_dir_cfg != "":  # coverage: ignore
-    cache_dir = Path(cache_dir_cfg)
 
-cache_expiration_cfg = config.get("cache", "expiration", fallback="180 days")
-cache_expiration = pd.Timedelta(cache_expiration_cfg)
+class Resolution(TypedDict, total=False):
+    category: str
+    name: str
+    environment_variable: str
+    default: str
+
+
+NAME_RESOLUTION: dict[str, Resolution] = {
+    # Cache configuration
+    "cache_dir": dict(
+        environment_variable="TRAFFIC_CACHE_PATH",
+        category="cache",
+        name="path",
+    ),
+    "cache_expiration": dict(
+        environment_variable="TRAFFIC_CACHE_EXPIRATION",
+        category="cache",
+        name="expiration",
+    ),
+    "aixm_path_str": dict(
+        environment_variable="TRAFFIC_AIXM_PATH",
+        category="global",
+        name="aixm_path",
+    ),
+    "nm_path_str": dict(
+        environment_variable="TRAFFIC_NM_PATH",
+        category="global",
+        name="nm_path",
+    ),
+    # Should we get a tqdm progress bar
+    "tqdm_style": dict(
+        environment_variable="TRAFFIC_TQDM_STYLE",
+        category="global",
+        name="tqdm_style",
+        default="auto",
+    ),
+    "aircraft_db": dict(
+        environment_variable="TRAFFIC_AIRCRAFTDB",
+        category="aircraft",
+        name="database",
+    ),
+}
 
-cache_purge_cfg = config.get("cache", "purge", fallback="")
-cache_no_expire = bool(os.environ.get("TRAFFIC_CACHE_NO_EXPIRE"))
 
-if cache_purge_cfg != "" and not cache_no_expire:  # coverage: ignore
-    cache_purge = pd.Timedelta(cache_purge_cfg)
-    now = pd.Timestamp("now").timestamp()
-
-    purgeable = list(
-        path
-        for path in cache_dir.glob("opensky/*")
-        if now - path.lstat().st_mtime > cache_purge.total_seconds()
-    )
-
-    if len(purgeable) > 0:
-        _log.warn(
-            f"Removing {len(purgeable)} cache files older than {cache_purge}"
-        )
-        for path in purgeable:
-            path.unlink()
+# Check the config file for a cache directory. If not present
+# then use the system default cache path
 
-if not cache_dir.exists():
-    cache_dir.mkdir(parents=True)
 
-# -- Tqdm Style Configuration --
-tqdm_style = config.get("global", "tqdm_style", fallback="auto")
+def get_config(
+    category: None | str = None,
+    name: None | str = None,
+    environment_variable: None | str = None,
+    default: None | str = None,
+) -> None | str:
+    if category is not None and name is not None:
+        if value := config.get(category, name, fallback=None):
+            return value
+
+    if environment_variable is not None:
+        return os.environ.get(environment_variable)
+
+    if default is not None:
+        return default
+
+    return None
+
+
+cache_dir = get_config(**NAME_RESOLUTION["cache_dir"])
+if cache_dir is None:
+    cache_dir = user_cache_dir("traffic")
+cache_path = Path(cache_dir)
+if not cache_path.exists():
+    cache_path.mkdir(parents=True)
+
+_cache_expiration_str = get_config(**NAME_RESOLUTION["cache_expiration"])
+cache_expiration = (
+    pd.Timedelta(_cache_expiration_str)
+    if _cache_expiration_str is not None
+    else None
+)
+_log.info(f"Selected cache_expiration: {cache_expiration}")
+
+aircraft_db_path = get_config(**NAME_RESOLUTION["aircraft_db"])
+_log.info(f"Selected aircraft_db path: {aircraft_db_path}")
+aixm_path_str = get_config(**NAME_RESOLUTION["aixm_path_str"])
+_log.info(f"Selected aixm path: {aixm_path_str}")
+nm_path_str = get_config(**NAME_RESOLUTION["nm_path_str"])
+_log.info(f"Selected nm path: {nm_path_str}")
+tqdm_style = get_config(**NAME_RESOLUTION["tqdm_style"])
 _log.info(f"Selected tqdm style: {tqdm_style}")
-
-# -- Plugin management --
-
-_enabled_plugins_raw = config.get("plugins", "enabled_plugins", fallback="")
-_enabled_list = ",".join(_enabled_plugins_raw.split("\n")).split(",")
-
-_selected = set(s.replace("-", "").strip().lower() for s in _enabled_list)
-_selected -= {""}
-
-_log.info(f"Selected plugins: {_selected}")
-
-if "TRAFFIC_NOPLUGIN" not in os.environ.keys():  # coverage: ignore
-    ep: Iterable[EntryPoint]
-    try:
-        # https://docs.python.org/3/library/importlib.metadata.html#entry-points
-        ep = entry_points(group="traffic.plugins")
-    except TypeError:
-        ep = entry_points().get("traffic.plugins", [])
-    for entry_point in ep:
-        name = entry_point.name.replace("-", "").lower()
-        if name in _selected:
-            _log.info(f"Loading plugin: {name}")
-            handle = entry_point.load()
-            _log.info(f"Loaded plugin: {handle.__name__}")
-            load = getattr(handle, "_onload", None)
-            if load is not None:
-                load()

src/traffic/algorithms/cpa.py

@@ -14,20 +14,21 @@
 import pandas as pd
 import pyproj
 
-from ..core import Flight, tqdm
 from ..core.mixins import DataFrameMixin
 
 if TYPE_CHECKING:
     from cartopy import crs
 
-    from ..core import Traffic
+    from ..core import Flight, Traffic
 
 _log = logging.getLogger(__name__)
 
 
 def combinations(
-    t: "Traffic", lateral_separation: float, vertical_separation: float
-) -> Iterator[Tuple[Flight, Flight]]:
+    t: "Traffic",
+    lateral_separation: float,
+    vertical_separation: float,
+) -> Iterator[Tuple["Flight", "Flight"]]:
     for flight in t:
         t_ = t.query(f'icao24 != "{flight.icao24}"')
         if t_ is None:
@@ -52,7 +53,9 @@ def combinations(
 
 class CPA(DataFrameMixin):
     def aggregate(
-        self, lateral_separation: float = 5, vertical_separation: float = 1000
+        self,
+        lateral_separation: float = 5,
+        vertical_separation: float = 1000,
     ) -> "CPA":
         return (
             self.assign(
@@ -172,7 +175,7 @@ def closest_point_of_approach(
     if isinstance(projection, crs.Projection):
         projection = pyproj.Proj(projection.proj4_init)
 
-    def yield_pairs(t_chunk: "Traffic") -> Iterator[Tuple[Flight, Flight]]:
+    def yield_pairs(t_chunk: "Traffic") -> Iterator[Tuple["Flight", "Flight"]]:
         """
         This function yields all pairs of possible candidates for a CPA
         calculation.
@@ -214,6 +217,8 @@ def yield_pairs(t_chunk: "Traffic") -> Iterator[Tuple[Flight, Flight]]:
     # TODO: it would probably be more efficient to multiprocess over each
     # t_chunk rather than multiprocess the distance computation.
 
+    from ..core import Flight, tqdm
+
     for _, t_chunk in tqdm(
         t_xyt.groupby("round_t"), total=len(set(t_xyt.data.round_t))
     ):