Support optional CASA columns (#270)

.github/workflows/ci.yml

@@ -44,11 +44,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Checkout source
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-
       - name: Cache Installations
         id: cache-installs
         uses: actions/cache@v3
@@ -103,8 +98,62 @@ jobs:
       #   if: ${{ failure() }}
       #   uses: mxschmitt/action-tmate@v3
 
+  test_apps:
+    needs: check_skip
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+      - name: Create Cache Hash
+        run: |
+          export HASH=$(sha256sum <<EOT | cut -c -16
+          ${{ env.POETRY_VERSION }}
+          EOT
+          )
+          echo "INSTALL_CACHE_HASH=$HASH" >> $GITHUB_ENV
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache Installations
+        id: cache-installs
+        uses: actions/cache@v3
+        with:
+          path: ~/.local
+          key: install-${{ env.INSTALL_CACHE_HASH }}-0
+
+      - name: Install Poetry
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 - --version ${{ env.POETRY_VERSION }}
+
+      - name: Test poetry run
+        run: poetry --version
+
+      - name: Checkout source
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Install dask-ms complete
+        run: poetry install --extras "testing complete"
+
+      - name: Test dask-ms applications
+        run: poetry run py.test -s -vvv --applications daskms/
+
+      # - name: Debug with tmate on failure
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
+
+
   deploy:
-    needs: [test]
+    needs: [test, test_apps]
     runs-on: ubuntu-latest
     # Run on a push to a tag or master
     if: >

HISTORY.rst

@@ -2,6 +2,10 @@
 History
 =======
 
+X.Y.Z (YYYY-MM-DD)
+------------------
+* Support optional CASA columns  (:pr:`270`)
+
 0.2.15 (2022-10-19)
 -------------------
 * Fix poetry install and cache hit detection on CI (:pr:`266`)

conftest.py

@@ -1,11 +1,4 @@
 # -*- coding: utf-8 -*-
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from os.path import join as pjoin
-
 collect_ignore = ["setup.py"]
 
 
@@ -24,24 +17,26 @@ def pytest_addoption(parser):
         default=False,
         help="Enable optional tests",
     )
+    parser.addoption(
+        "--applications",
+        action="store_true",
+        dest="applications",
+        default=False,
+        help="Enable application tests",
+    )
 
 
 def pytest_configure(config):
     # Add non-standard markers
     config.addinivalue_line("markers", "stress: long running stress tests")
     config.addinivalue_line("markers", "optional: optional tests")
+    config.addinivalue_line("markers", "applications: application tests")
 
-    # Enable/disable them based on parsed config
-    disable_str = []
-
-    if not config.option.stress:
-        disable_str.append("not stress")
-
-    if not config.option.optional:
-        disable_str.append("not optional")
+    markexpr = [config.option.markexpr] if config.option.markexpr else []
 
-    disable_str = " and ".join(disable_str)
+    for mark in ("stress", "optional", "applications"):
+        test = "" if getattr(config.option, mark, False) else "not "
+        markexpr.append(f"{test}{mark}")
 
-    if disable_str != "":
-        print(disable_str)
-        setattr(config.option, "markexpr", disable_str)
+    config.option.markexpr = " and ".join(markexpr)
+    print(config.option.markexpr)

daskms/apps/conftest.py

@@ -0,0 +1,67 @@
+from appdirs import user_cache_dir
+from hashlib import sha256
+import logging
+from pathlib import Path
+import tarfile
+
+import pytest
+
+log = logging.getLogger(__file__)
+
+TAU_MS = "HLTau_B6cont.calavg.tav300s"
+TAU_MS_TAR = f"{TAU_MS}.tar.xz"
+TAU_MS_TAR_HASH = "fc2ce9261817dfd88bbdd244c8e9e58ae0362173938df6ef2a587b1823147f70"
+DATA_URL = f"s3://ratt-public-data/test-data/{TAU_MS_TAR}"
+
+
+def download_tau_ms(tau_ms_tar):
+    if tau_ms_tar.exists():
+        with open(tau_ms_tar, "rb") as f:
+            digest = sha256()
+
+            while data := f.read(2**20):
+                digest.update(data)
+
+            if digest.hexdigest() == TAU_MS_TAR_HASH:
+                return
+
+            tau_ms_tar.unlink(missing_ok=True)
+            raise ValueError(
+                f"sha256 digest '{digest.hexdigest()}' "
+                f"of {tau_ms_tar} does not match "
+                f"{TAU_MS_TAR_HASH}"
+            )
+    else:
+        s3fs = pytest.importorskip("s3fs")
+        s3 = s3fs.S3FileSystem(anon=True)
+
+        for attempt in range(3):
+            with s3.open(DATA_URL, "rb") as fin, open(tau_ms_tar, "wb") as fout:
+                digest = sha256()
+
+                while data := fin.read(2**20):
+                    digest.update(data)
+                    fout.write(data)
+
+                if digest.hexdigest() == TAU_MS_TAR_HASH:
+                    return
+
+                log.warning("Download of %s failed on attempt %d", DATA_URL, attempt)
+                tau_ms_tar.unlink(missing_ok=True)
+
+        raise ValueError(f"Download of {DATA_URL} failed {attempt} times")
+
+
+@pytest.fixture(scope="function")
+def tau_ms(tmp_path_factory):
+    cache_dir = Path(user_cache_dir("dask-ms")) / "test-data"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    tau_ms_tar = cache_dir / TAU_MS_TAR
+
+    download_tau_ms(tau_ms_tar)
+    msdir = tmp_path_factory.mktemp("taums")
+
+    with tarfile.open(tau_ms_tar) as tar:
+        tar.extractall(msdir)
+
+    yield msdir / TAU_MS