Add a CI test for the dbt-metricflow package (#1653)

This PR adds a test in CI that checks if the `dbt-metricflow` package
builds properly. The check is accomplished by running the tutorial and
then running the sample query.

.github/workflows/ci-metricflow-unit-tests.yaml

@@ -93,4 +93,5 @@ jobs:
             }
 
       - name: Run Package-Build Tests
+        shell: bash
         run: "make test-build-packages"

scripts/ci_tests/dbt_metricflow_package_test.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import subprocess
+import textwrap
+from pathlib import Path
+from typing import Optional
+
+
+def _run_shell_command(command: str, cwd: Optional[Path] = None) -> None:
+    if cwd is None:
+        cwd = Path.cwd()
+
+    print(
+        textwrap.dedent(
+            f"""\
+            Running via shell:
+                command: {command!r}
+                cwd: {cwd.as_posix()!r}
+            """
+        ).rstrip()
+    )
+    subprocess.check_call(command, shell=True, cwd=cwd.as_posix())
+
+
+if __name__ == "__main__":
+    # Check that the `mf` command is installed.
+    _run_shell_command("which python")
+    _run_shell_command("which mf")
+    _run_shell_command("mf")
+    # Run the tutorial using `--yes` to create the sample project without user interaction.
+    _run_shell_command("mf tutorial --yes")
+    tutorial_directory = Path.cwd().joinpath("mf_tutorial_project")
+
+    # Run the first few tutorial steps.
+    _run_shell_command("dbt seed", cwd=tutorial_directory)
+    _run_shell_command("dbt build", cwd=tutorial_directory)
+    _run_shell_command(
+        "mf query --metrics transactions --group-by metric_time --order metric_time",
+        cwd=tutorial_directory,
+    )

scripts/ci_tests/run_package_build_tests.py

@@ -8,14 +8,17 @@
 import logging
 import tempfile
 import venv
+from collections.abc import Sequence
 from pathlib import Path
 
 from scripts.mf_script_helper import MetricFlowScriptHelper
 
 logger = logging.getLogger(__name__)
 
 
-def _run_package_build_test(package_directory: Path, package_test_script: Path) -> None:
+def _run_package_build_test(
+    package_directory: Path, package_test_script: Path, optional_package_dependencies_to_install: Sequence[str] = ()
+) -> None:
     """Run a test to verify that a package is built properly.
 
     Given the directory where the package is located, this will build the package using `hatch build` and install the
@@ -25,40 +28,62 @@ def _run_package_build_test(package_directory: Path, package_test_script: Path)
     Args:
         package_directory: Root directory where the package is located.
         package_test_script: The path to the script that should be run.
-
+        optional_package_dependencies_to_install: If the given package defines optional dependencies that can be
+        installed, install these. e.g. for `dbt-metricflow[dbt-duckdb]`, specify `dbt-duckdb`.
     Returns: None
     Raises: Exception on test failure.
     """
-    logger.info(f"Running package build test for {str(package_directory)!r} using {str(package_test_script)!r}")
+    package_directory_str = package_directory.as_posix()
+    package_test_script_str = package_test_script.as_posix()
+    logger.info(f"Running package build test for {package_directory_str!r} using {package_test_script_str!r}")
+
     try:
         with tempfile.TemporaryDirectory() as temporary_directory_str:
             temporary_directory = Path(temporary_directory_str)
             venv_directory = temporary_directory.joinpath("venv")
-            logger.info(f"Creating venv at {str(venv_directory)!r}")
+            logger.info(f"Creating a new venv at {venv_directory.as_posix()!r}")
 
             venv.create(venv_directory, with_pip=True)
-            pip_executable = Path(venv_directory, "bin/pip")
-            python_executable = Path(venv_directory, "bin/python")
+            pip_executable = Path(venv_directory, "bin/pip").as_posix()
 
-            logger.info(f"Building package at {str(package_directory)!r}")
-            logger.info(f"Running package build test for {str(package_directory)!r} using {str(package_test_script)!r}")
+            logger.info(f"Building package at {package_directory_str!r}")
+            MetricFlowScriptHelper.run_command(["hatch", "clean"], working_directory=package_directory)
             MetricFlowScriptHelper.run_command(["hatch", "build"], working_directory=package_directory)
 
-            logger.info("Installing package using generated wheels")
-            MetricFlowScriptHelper.run_shell_command(f'{pip_executable} install "{str(package_directory)}"/dist/*.whl')
-
-            logger.info("Running test using installed package in venv")
-            MetricFlowScriptHelper.run_command(
-                [str(python_executable), str(package_test_script)], working_directory=temporary_directory
+            logger.info("Installing package in venv using generated wheels")
+            paths_to_wheels = _get_wheels_in_directory(package_directory.joinpath("dist"))
+            if len(paths_to_wheels) != 1:
+                raise RuntimeError(f"Expected exactly one wheel but got {paths_to_wheels}")
+
+            path_to_wheel = paths_to_wheels[0]
+            MetricFlowScriptHelper.run_command([pip_executable, "install", path_to_wheel.as_posix()])
+            for optional_package_dependency in optional_package_dependencies_to_install:
+                MetricFlowScriptHelper.run_command(
+                    [pip_executable, "install", f"{path_to_wheel.as_posix()}[{optional_package_dependency}]"]
+                )
+
+            logger.info("Running test using venv")
+            venv_activate = venv_directory.joinpath("bin", "activate").as_posix()
+            MetricFlowScriptHelper.run_shell_command(
+                # Using period instead of `source` for compatibility with `sh`.
+                f"cd {temporary_directory_str} && . {venv_activate} && python {package_test_script_str}",
             )
 
-            logger.info(f"Test passed {str(package_test_script)!r}")
+            logger.info(f"Test passed {package_test_script_str!r}")
     except Exception as e:
         raise PackageBuildTestFailureException(
-            f"Package build test failed for {str(package_directory)!r} using {str(package_test_script)!r}"
+            f"Package build test failed for {package_directory_str!r} using {package_test_script_str!r}"
         ) from e
 
 
+def _get_wheels_in_directory(directory: Path) -> Sequence[Path]:
+    paths_to_wheels = []
+    for path_item in directory.iterdir():
+        if path_item.is_file() and path_item.suffix == ".whl":
+            paths_to_wheels.append(path_item)
+    return paths_to_wheels
+
+
 class PackageBuildTestFailureException(Exception):  # noqa: D101
     pass
 
@@ -80,10 +105,15 @@ class PackageBuildTestFailureException(Exception):  # noqa: D101
         package_test_script=metricflow_repo_directory.joinpath("scripts/ci_tests/metricflow_package_test.py"),
     )
 
-    # Test building `metricflow-semantics` package.
+    # Test building the `metricflow-semantics` package.
     _run_package_build_test(
         package_directory=metricflow_repo_directory.joinpath("metricflow-semantics"),
         package_test_script=metricflow_repo_directory.joinpath("scripts/ci_tests/metricflow_semantics_package_test.py"),
     )
 
-    # Add entry for `dbt-metricflow` once build issues are resolved.
+    # Test building the `dbt-metricflow` package.
+    _run_package_build_test(
+        package_directory=metricflow_repo_directory.joinpath("dbt-metricflow"),
+        package_test_script=metricflow_repo_directory.joinpath("scripts/ci_tests/dbt_metricflow_package_test.py"),
+        optional_package_dependencies_to_install=("dbt-duckdb",),
+    )