From 6df3421299e59feb77b70684cbc187940947f722 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 16:14:53 -0500 Subject: [PATCH 01/62] move config files into pyproject.toml and hatch.toml --- .bumpversion.cfg | 35 -------------- test.env.example => .env.example | 0 MANIFEST.in | 1 - Makefile | 48 ------------------- dev-requirements.txt | 23 --------- hatch.toml | 61 ++++++++++++++++++++++++ pyproject.toml | 52 ++++++++++++++++++++ pytest.ini | 10 ---- setup.py | 81 -------------------------------- tox.ini | 32 ------------- 10 files changed, 113 insertions(+), 230 deletions(-) delete mode 100644 .bumpversion.cfg rename test.env.example => .env.example (100%) delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 hatch.toml create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 7ff98322d..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.9.0b1 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/snowflake/__version__.py] diff --git a/test.env.example b/.env.example similarity index 100% rename from test.env.example rename to .env.example diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index 9a2376f47..000000000 --- a/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-snowflake - -.PHONY: unit -unit: ## Runs unit tests with py39. - @\ - tox -e py39 - -.PHONY: test -test: ## Runs unit tests with py39 and code checks against staged changes. - @\ - tox -p -e py39; \ - pre-commit run --all-files - -.PHONY: integration -integration: ## Runs snowflake integration tests with py38. - @\ - tox -e py39-snowflake -- - -.PHONY: clean - @echo "cleaning repo" - @git clean -f -X - -.PHONY: help -help: ## Show this help message. - @echo 'usage: make [target]' - @echo - @echo 'targets:' - @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -.PHONY: docker-dev -docker-dev: - docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . - docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev - -.PHONY: docker-prod -docker-prod: - docker build -f docker/Dockerfile -t dbt-snowflake . diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 906003768..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# install latest changes in dbt-core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git - -# dev -ipdb~=0.13.13 -pre-commit~=3.7.0 - -# test -ddtrace==2.3.0 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.6 -tox~=4.16 - -# build -bumpversion~=0.6.0 -twine~=5.1 -wheel~=0.43 diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 000000000..01465cbc0 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,61 @@ +[version] +path = "dbt/adapters/snowflake/__version__.py" + +[build.targets.sdist] +packages = ["dbt"] + +[build.targets.wheel] +packages = ["dbt"] + +[envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "ddtrace==2.3.0", + "ipdb~=0.13.13", + "pre-commit==3.7.0", + "freezegun", + "pytest", + "pytest-csv~=3.0", + "pytest-dotenv", + "pytest-logbook~=1.2", + "pytest-mock", + "pytest-xdist", +] + +[envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "python -m pytest {args:tests/functional}" +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", + "docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev", +] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] + +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt_snowflake-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-snowflake", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_snowflake-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-snowflake", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-snowflake ." diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..577932f47 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-snowflake" +description = "The Snowflake adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "snowflake"] +requires-python = ">=3.9.0" +authors = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +maintainers = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.7,<2.0", + "snowflake-connector-python[secure-local-storage]~=3.0", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0", + # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "agate", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-snowflake" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-snowflake.git" +Issues = "https://github.com/dbt-labs/dbt-snowflake/issues" +Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" + +[tool.pytest] +testpaths = ["tests/functional", "tests/unit"] +color = true +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b04a6ccf3..000000000 --- a/pytest.ini +++ /dev/null @@ -1,10 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/integration - tests/functional diff --git a/setup.py b/setup.py deleted file mode 100644 index c0716341d..000000000 --- a/setup.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -import os -from pathlib import Path - -import sys - -# require python 3.8 or newer -if sys.version_info < (3, 9): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.9 or higher.") - sys.exit(1) - - -# require version of setuptools that supports find_namespace_packages -from setuptools import setup - -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again") - sys.exit(1) - - -# pull long description from README -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, "README.md")) as f: - long_description = f.read() - - -# used for this adapter's version -VERSION = Path(__file__).parent / "dbt/adapters/snowflake/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -package_name = "dbt-snowflake" -description = """The Snowflake adapter plugin for dbt""" - -setup( - name=package_name, - version=_plugin_version(), - description=description, - long_description=long_description, - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-snowflake", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", - "snowflake-connector-python[secure-local-storage]~=3.0", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0", - # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "agate", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - ], - python_requires=">=3.9", -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index f6952efaf..000000000 --- a/tox.ini +++ /dev/null @@ -1,32 +0,0 @@ -[tox] -skipsdist = True -envlist = py39,py310,py311,py312 - -[testenv:{unit,py39,py310,py311,py312,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - SNOWFLAKE_TEST_* - PYTEST_ADDOPTS - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - snowflake: {envpython} -m pytest {posargs} tests/functional -deps = - -rdev-requirements.txt - -e. From 58694133000429be45ce564c6a04a468d6a15cf2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 17:44:27 -0500 Subject: [PATCH 02/62] update workflow files to use hatch.toml --- .github/scripts/update_dependencies.sh | 6 +- .../scripts/update_dev_dependency_branches.sh | 4 +- .github/workflows/integration.yml | 88 +--- .github/workflows/main.yml | 124 +---- .github/workflows/release.yml | 113 ++--- .github/workflows/release_prep_hatch.yml | 469 ++++++++++++++++++ pyproject.toml | 1 + 7 files changed, 555 insertions(+), 250 deletions(-) create mode 100644 .github/workflows/release_prep_hatch.yml diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh index c3df48e52..7880b0163 100644 --- a/.github/scripts/update_dependencies.sh +++ b/.github/scripts/update_dependencies.sh @@ -2,9 +2,9 @@ set -e git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" +tests_req_sed_pattern="s|dbt-adapters.git#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index 022df6a8a..c78029e5e 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -5,8 +5,8 @@ set -e dbt_adapters_branch=$1 dbt_core_branch=$2 dbt_common_branch=$3 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b3662d5c0..1ff7c1149 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -86,23 +86,17 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.generate-matrix.outputs.result }} - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' + - if: github.event_name != 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' + - if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' + - if: github.event_name == 'pull_request_target' # https://github.com/marketplace/actions/paths-changes-filter # For each filter, it sets output variable named by the filter to the text: # 'true' - if any of changed files matches any of filter rules @@ -119,10 +113,9 @@ jobs: - '.github/**/*.sh' - 'dbt/**' - 'tests/**' - - 'dev-requirements.txt' + - 'pyproject.toml' - '*.py' - - name: Generate integration test matrix - id: generate-matrix + - id: generate-matrix uses: actions/github-script@v7 env: CHANGES: ${{ steps.get-changes.outputs.changes }} @@ -132,9 +125,9 @@ jobs: const matrix = script({ context }) console.log(matrix) return matrix + test: name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - # run if not a PR from a forked repository or has a label to mark as safe to test # also checks that the matrix generated is not empty if: >- @@ -147,67 +140,42 @@ jobs: ) runs-on: ${{ matrix.os }} needs: test-metadata - strategy: fail-fast: false matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} - steps: - - name: Check out the repository - if: github.event_name == 'push' + - if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false - - - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' + - if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ inputs.adapter_branch }} - - # explicitly checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' + - if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Update Adapters and Core branches (update dev_requirements.txt) - if: ${{ github.event_name == 'workflow_dispatch' }} + - if: ${{ github.event_name == 'workflow_dispatch' }} run: | ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} - cat dev-requirements.txt - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox (snowflake) - if: matrix.adapter == 'snowflake' + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - uses: pypa/hatch@install + - run: hatch run integration-tests env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} @@ -226,38 +194,14 @@ jobs: DBT_TEST_USER_1: dbt_test_role_1 DBT_TEST_USER_2: dbt_test_role_2 DBT_TEST_USER_3: dbt_test_role_3 - run: tox -- --ddtrace - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }} - path: ./logs - overwrite: true - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - overwrite: true require-label-comment: runs-on: ubuntu-latest - needs: test - permissions: pull-requests: write - steps: - - name: Needs permission PR comment - if: >- + - if: >- needs.test.result == 'skipped' && github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24d2fa60b..d89f635d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,124 +38,52 @@ defaults: jobs: code-quality: name: code-quality - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - dbt --version - - - name: Run pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure + - uses: pre-commit/action@v3.0.1 unit: name: unit test / python ${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - overwrite: true + - uses: pypa/hatch@install + - run: hatch run unit-tests build: name: build packages - runs-on: ubuntu-latest - outputs: is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} - steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: Build distributions - run: ./scripts/build-dist.sh - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: Check if this is an alpha version - id: check-is-alpha + - run: hatch build + - run: hatch run build:check-all + - id: check-is-alpha run: | export is_alpha=0 if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v4 with: name: dist @@ -164,44 +92,22 @@ jobs: test-build: name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} - if: needs.build.outputs.is_alpha == 0 - needs: build - runs-on: ${{ matrix.os }} - strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-14, windows-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - uses: actions/download-artifact@v4 with: name: dist path: dist/ - - - name: Show distributions - run: ls -lh dist/ - - - name: Install ${{ matrix.dist-type }} distributions - run: | - find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check ${{ matrix.dist-type }} distributions - run: | - python -c "import dbt.adapters.snowflake" + - run: find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + - run: python -c "import dbt.adapters.snowflake" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad7cf76b4..771228d73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,86 +119,71 @@ defaults: shell: bash jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Only Docker: ${{ inputs.only_docker }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + release-prep: + name: "Release prep: generate changelog, bump version" + uses: dbt-labs/dbt-snowflake/.github/workflows/release_prep_hatch.yml@main with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] + build-release: + name: "Build release" + needs: release-prep runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} steps: - - name: Print variables + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-prep.outputs.release-branch }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - uses: pypa/hatch@install + - id: archive run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + - run: hatch build && hatch run build:check-all + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.archive.outputs.name }} + path: dist/ + retention-days: 3 - build-test-package: - name: Build, Test, Package + pypi-release: + name: "PyPI release" if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + runs-on: ubuntu-latest + needs: build-release + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + permissions: + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write + steps: + - name: "Publish to PyPI" + uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ needs.build-release.outputs.archive-name }} github-release: name: GitHub Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + needs: [build-release, release-prep] + uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + archive_name: ${{ needs.build-release.outputs.archive-name }} docker-release: name: "Docker Release" @@ -206,7 +191,7 @@ jobs: # what we need to release but draft releases don't actually tag the commit so it # finds nothing to release if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} - needs: [bump-version-generate-changelog, build-test-package, github-release] + needs: github-release permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml new file mode 100644 index 000000000..18d7637b8 --- /dev/null +++ b/.github/workflows/release_prep_hatch.yml @@ -0,0 +1,469 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# branch: The branch that we will release from +# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# deploy-to: If we are deploying to prod or test, if test then release from branch +# is-nightly-release: Identifier that this is nightly release +# +# Outputs: +# release-sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +name: "Release prep" +run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +on: + workflow_call: + inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The version to release" + required: true + type: string + deploy-to: + description: "Deploy to test or prod" + type: string + default: "prod" + is-nightly-release: + description: "Identify if this is a nightly release" + type: boolean + default: false + outputs: + release-branch: + description: "The branch to be released from" + value: ${{ jobs.release.outputs.branch }} + release-sha: + description: "The SHA to be released" + value: ${{ jobs.release.outputs.sha }} + changelog-path: + description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" + value: ${{ jobs.release-inputs.outputs.changelog-path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.12 + NOTIFICATION_PREFIX: "[Release Prep]" + +jobs: + release-inputs: + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.changelog.outputs.path }} + changelog-exists: ${{ steps.changelog.outputs.exists }} + base-version: ${{ steps.semver.outputs.base-version }} + pre-release: ${{ steps.semver.outputs.pre-release }} + is-pre-release: ${{ steps.semver.outputs.is-pre-release }} + version-is-current: ${{ steps.version.outputs.is-current }} + + steps: + - name: "[DEBUG] Log inputs" + run: | + # WORKFLOW INPUTS + echo Branch: ${{ inputs.branch }} + echo Release version: ${{ inputs.version }} + echo Deploy to: ${{ inputs.deploy-to }} + echo Nightly release: ${{ inputs.is-nightly-release }} + # ENVIRONMENT VARIABLES + echo Python version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Parse input version" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ inputs.version }} + + - name: "Audit version" + id: version + run: | + is_current=false + current_version=$(hatch version) + if test "$current_version" = "${{ inputs.version }}" + then + is_current=true + fi + echo "is-current=$is_current" >> $GITHUB_OUTPUT + + - name: "[INFO] Skip version bump" + if: steps.version.outputs.is-current == 'true' + run: | + title="Skip version bump" + message="The version matches the input version ${{ inputs.version }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Audit changelog" + id: changelog + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + does_exist=false + if test -f $path + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[INFO] Skip changelog generation" + if: steps.changelog.outputs.exists == 'true' + run: | + title="Skip changelog generation" + message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-branch: + runs-on: ubuntu-latest + needs: release-inputs + if: | + needs.release-inputs.outputs.changelog-exists == 'false' || + needs.release-inputs.outputs.version-is-current == 'false' + outputs: + name: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Set release branch" + id: release-branch + run: | + name="prep-release/" + if [[ ${{ inputs.is-nightly-release }} == true ]] + then + name+="nightly-release/" + else + name+="${{ inputs.deploy-to }}/" + fi + name+="${{ inputs.version }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Create release branch ${{ steps.release-branch.outputs.name }}" + run: | + git checkout -b ${{ steps.release-branch.outputs.name }} + git push -u origin ${{ steps.release-branch.outputs.name }} + + - name: "[INFO] Create release branch" + run: | + title="Create release branch" + message="Create release branch: ${{ steps.release-branch.outputs.name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + core-team: + if: needs.release-inputs.outputs.changelog-exists == 'false' + needs: release-inputs + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + secrets: inherit + + generate-changelog: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.changelog-exists == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - core-team + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Install `changie`" + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --move-dir '${{ needs.release-inputs.outputs.base-version }}' \ + --prerelease ${{ needs.release-inputs.outputs.pre-release }} + elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --include '${{ needs.release-inputs.outputs.base-version }}' \ + --remove-prereleases + else # releasing a final patch with no pre-releases + changie batch ${{ needs.release-inputs.outputs.base-version }} + fi + changie merge + env: + CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }} + + - name: "Remove trailing whitespace and missing new lines" + # this step will fail on whitespace errors but also correct them + continue-on-error: true + run: hatch run code-quality + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + + - name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + title="Changelog generation" + if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]] + then + message="Generated changelog file successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Failed to generate changelog file" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + bump-version: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.version-is-current == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - generate-changelog + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Bump version to ${{ inputs.version }}" + run: hatch version ${{ inputs.version }} + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version to ${{ inputs.version }}" + run: | + title="Version bump" + message="Bumped version to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + unit-tests: + runs-on: ubuntu-latest + # only run unit tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run unit tests" + run: hatch run unit-tests + + integration-tests: + runs-on: ubuntu-latest + # only run integration tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: Setup postgres + run: psql -f ./scripts/setup_test_database.sql + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + + - name: "Set up `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run integration tests" + run: hatch run integration-tests + env: + POSTGRES_TEST_HOST: localhost + POSTGRES_TEST_PORT: 5432 + POSTGRES_TEST_USER: root + POSTGRES_TEST_PASS: password + POSTGRES_TEST_DATABASE: dbt + POSTGRES_TEST_THREADS: 4 + + merge-release-branch: + runs-on: ubuntu-latest + needs: + - unit-tests + - integration-tests + - release-branch + - release-inputs + if: | + !failure() && !cancelled() && + needs.release-branch.result == 'success' && + inputs.deploy-to == 'prod' + + steps: + - name: "Checkout ${{ github.event.repository.name }}" + uses: actions/checkout@v4 + + - name: "Merge changes into ${{ inputs.branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.release-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[INFO] Merge changes into ${{ inputs.branch }}" + run: | + title="Merge changes" + message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release: + runs-on: ubuntu-latest + needs: + - release-branch + - merge-release-branch + if: ${{ !failure() && !cancelled() }} + + # Get the SHA that will be released. + # If the changelog already exists and the version was already current on the input branch, then release from there. + # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + branch: ${{ steps.branch.outputs.name }} + sha: ${{ steps.sha.outputs.sha }} + + steps: + - name: "Set release branch" + id: branch + # If a release branch was created and not merged, use the release branch + # Otherwise, use the input branch because either nothing was done, or the changes were merged back in + run: | + if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then + branch="${{ needs.release-branch.outputs.name }}" + else + branch="${{ inputs.branch }}" + fi + echo "name=$branch" >> $GITHUB_OUTPUT + + - name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.branch.outputs.name }} + + - name: "Set release SHA" + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # if this is a real release and a release branch was created, delete it + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} diff --git a/pyproject.toml b/pyproject.toml index 577932f47..520e21a51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" [tool.pytest] testpaths = ["tests/functional", "tests/unit"] +addopts = "-v -n auto" color = true filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", From 06b5c336b2e32589f065734eaa385541db0b6e5e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:02:01 -0500 Subject: [PATCH 03/62] revert formatting changes in integration.yml --- .github/workflows/integration.yml | 42 +++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1ff7c1149..a8c7f1137 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -86,17 +86,23 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.generate-matrix.outputs.result }} + steps: - - if: github.event_name != 'pull_request_target' + - name: Check out the repository (non-PR) + if: github.event_name != 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false - - if: github.event_name == 'pull_request_target' + + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - if: github.event_name == 'pull_request_target' + + - name: Check if relevant files changed + if: github.event_name == 'pull_request_target' # https://github.com/marketplace/actions/paths-changes-filter # For each filter, it sets output variable named by the filter to the text: # 'true' - if any of changed files matches any of filter rules @@ -115,7 +121,8 @@ jobs: - 'tests/**' - 'pyproject.toml' - '*.py' - - id: generate-matrix + - name: Generate integration test matrix + id: generate-matrix uses: actions/github-script@v7 env: CHANGES: ${{ steps.get-changes.outputs.changes }} @@ -140,9 +147,11 @@ jobs: ) runs-on: ${{ matrix.os }} needs: test-metadata + strategy: fail-fast: false matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} + env: DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true @@ -150,31 +159,45 @@ jobs: DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} + steps: - - if: github.event_name == 'push' + - name: Check out the repository + if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false - - if: github.event_name == 'workflow_dispatch' + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ inputs.adapter_branch }} - - if: github.event_name == 'pull_request_target' + + # explicitly checkout the branch for the PR, + # this is necessary for the `pull_request_target` event + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - if: ${{ github.event_name == 'workflow_dispatch' }} + + - name: Set up Python ${{ matrix.python-version }} + if: ${{ github.event_name == 'workflow_dispatch' }} run: | ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} + cat hatch.toml + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: pypa/hatch@install + - run: hatch run integration-tests env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} @@ -197,9 +220,12 @@ jobs: require-label-comment: runs-on: ubuntu-latest + needs: test + permissions: pull-requests: write + steps: - if: >- needs.test.result == 'skipped' && From 890c42dda3aad242bda8a554b8bfac3ce0a3e0f8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:04:59 -0500 Subject: [PATCH 04/62] revert formatting changes in integration.yml --- .github/workflows/integration.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a8c7f1137..0745fdc8b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -132,9 +132,9 @@ jobs: const matrix = script({ context }) console.log(matrix) return matrix - test: name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} + # run if not a PR from a forked repository or has a label to mark as safe to test # also checks that the matrix generated is not empty if: >- @@ -184,6 +184,11 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Update Adapters and Core branches (update hatch.toml) if: ${{ github.event_name == 'workflow_dispatch' }} run: | ./.github/scripts/update_dev_dependency_branches.sh \ @@ -227,7 +232,8 @@ jobs: pull-requests: write steps: - - if: >- + - name: Needs permission PR comment + if: >- needs.test.result == 'skipped' && github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository From a3b8287b862661aeeb837bbf807d9024046d6d2a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:13:22 -0500 Subject: [PATCH 05/62] revert formatting changes in main.yml --- .github/workflows/integration.yml | 1 + .github/workflows/main.yml | 67 +++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0745fdc8b..026e980ad 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -119,6 +119,7 @@ jobs: - '.github/**/*.sh' - 'dbt/**' - 'tests/**' + - 'hatch.toml' - 'pyproject.toml' - '*.py' - name: Generate integration test matrix diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d89f635d8..c35a0105f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,52 +38,81 @@ defaults: jobs: code-quality: name: code-quality + runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: '3.9' - - uses: pre-commit/action@v3.0.1 + + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.1 unit: name: unit test / python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] + steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: pypa/hatch@install - run: hatch run unit-tests build: name: build packages + runs-on: ubuntu-latest + outputs: is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} + steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@v4 with: persist-credentials: false - - uses: actions/setup-python@v5 + + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: '3.9' - - run: hatch build - - run: hatch run build:check-all - - id: check-is-alpha + + - uses: pypa/hatch@install + + - name: Build distributions + run: hatch build + + - name: Check distribution descriptions + run: hatch run build:check-all + + - name: Check if this is an alpha version + id: check-is-alpha run: | export is_alpha=0 if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v4 with: name: dist @@ -92,22 +121,34 @@ jobs: test-build: name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} + if: needs.build.outputs.is_alpha == 0 + needs: build + runs-on: ${{ matrix.os }} + strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-14, windows-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] + steps: - - uses: actions/setup-python@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v4 with: name: dist path: dist/ - - run: find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - run: python -c "import dbt.adapters.snowflake" + + - name: Install ${{ matrix.dist-type }} distributions + run: find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check ${{ matrix.dist-type }} distributions + run: | + python -c "import dbt.adapters.snowflake" From a63cbb7dbadb39a4904e71c20fc5f601d3f80a46 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:29:17 -0500 Subject: [PATCH 06/62] revert formatting changes in main.yml --- .github/workflows/main.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c35a0105f..da398b9ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - - uses: pypa/hatch@install + - name: Install python dependencies + uses: pypa/hatch@install + - run: hatch run unit-tests build: @@ -103,6 +105,9 @@ jobs: - name: Build distributions run: hatch build + - name: Show distribution contents + run: ls -lh dist/ + - name: Check distribution descriptions run: hatch run build:check-all @@ -146,8 +151,12 @@ jobs: name: dist path: dist/ + - name: Show distributions + run: ls -lh dist/ + - name: Install ${{ matrix.dist-type }} distributions - run: find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + run: | + find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - name: Check ${{ matrix.dist-type }} distributions run: | From 9f2f504b52f002935fe593feee2120e5c68c79c3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:32:33 -0500 Subject: [PATCH 07/62] revert formatting changes in release.yml --- .github/workflows/release.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 771228d73..e503603cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -154,6 +154,18 @@ jobs: path: dist/ retention-days: 3 + github-release: + name: GitHub Release + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} + needs: [build-release, release-prep] + uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main + with: + sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + version_number: ${{ inputs.version_number }} + changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + test_run: ${{ inputs.test_run }} + archive_name: ${{ needs.build-release.outputs.archive-name }} + pypi-release: name: "PyPI release" if: ${{ !failure() && !cancelled() && !inputs.only_docker }} @@ -173,18 +185,6 @@ jobs: repository-url: ${{ vars.PYPI_REPOSITORY_URL }} archive-name: ${{ needs.build-release.outputs.archive-name }} - github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [build-release, release-prep] - uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.test_run }} - archive_name: ${{ needs.build-release.outputs.archive-name }} - docker-release: name: "Docker Release" # We cannot release to docker on a test run because it uses the tag in GitHub as From 4819e5e2f0036ea16d0be968784b43efcb7ae286 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:34:41 -0500 Subject: [PATCH 08/62] revert formatting changes in release.yml --- .github/workflows/release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e503603cb..4653f4e84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -167,7 +167,7 @@ jobs: archive_name: ${{ needs.build-release.outputs.archive-name }} pypi-release: - name: "PyPI release" + name: PyPI Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} runs-on: ubuntu-latest needs: build-release @@ -179,8 +179,7 @@ jobs: # see https://github.com/marketplace/actions/pypi-publish id-token: write steps: - - name: "Publish to PyPI" - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main + - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main with: repository-url: ${{ vars.PYPI_REPOSITORY_URL }} archive-name: ${{ needs.build-release.outputs.archive-name }} From 16f0f06a988d4eb4a077ae673493ed6c0b8fcd2f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:35:32 -0500 Subject: [PATCH 09/62] revert formatting changes in release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4653f4e84..18fa346e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -169,8 +169,8 @@ jobs: pypi-release: name: PyPI Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - runs-on: ubuntu-latest needs: build-release + runs-on: ubuntu-latest environment: name: ${{ inputs.deploy-to }} url: ${{ vars.PYPI_PROJECT_URL }} From 85114b1fdbbd0d8052d7b75bc84cdbf65c13a647 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 18:44:40 -0500 Subject: [PATCH 10/62] changelog --- .changes/unreleased/Under the Hood-20241117-184430.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20241117-184430.yaml diff --git a/.changes/unreleased/Under the Hood-20241117-184430.yaml b/.changes/unreleased/Under the Hood-20241117-184430.yaml new file mode 100644 index 000000000..d2a7b67a6 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241117-184430.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move from setup.py to pyproject.toml and to hatch as a dev tool +time: 2024-11-17T18:44:30.85288-05:00 +custom: + Author: mikealfare + Issue: "1250" From 38198e9c5e67ac1d16c2e116b1308c62dd346dd3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:36:14 -0500 Subject: [PATCH 11/62] add dd tracking --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 026e980ad..b9fad5096 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -204,7 +204,7 @@ jobs: - uses: pypa/hatch@install - - run: hatch run integration-tests + - run: hatch run integration-tests --ddtrace env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 00d88dd9dd52d49da4d628180e87cb4b8e35e1d3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 19:52:12 -0500 Subject: [PATCH 12/62] remove references to postgres --- .github/workflows/release_prep_hatch.yml | 31 +----------------------- 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 18d7637b8..de782ad1a 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -355,46 +355,17 @@ jobs: - generate-changelog - bump-version - services: - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" uses: actions/checkout@v4 with: - ref: ${{ needs.release-branch.outputs.name }} - - - name: Setup postgres - run: psql -f ./scripts/setup_test_database.sql - env: - PGHOST: localhost - PGPORT: 5432 - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres + ref: ${{ needs.release-branch.outputs.name }}\ - name: "Set up `hatch`" uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Run integration tests" run: hatch run integration-tests - env: - POSTGRES_TEST_HOST: localhost - POSTGRES_TEST_PORT: 5432 - POSTGRES_TEST_USER: root - POSTGRES_TEST_PASS: password - POSTGRES_TEST_DATABASE: dbt - POSTGRES_TEST_THREADS: 4 merge-release-branch: runs-on: ubuntu-latest From 582afbfc72440a8793eb68f22504bf204317921b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 20:48:00 -0500 Subject: [PATCH 13/62] simplify pyproject.toml --- pyproject.toml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 520e21a51..9cb49cc93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,8 @@ description = "The Snowflake adapter plugin for dbt" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "snowflake"] requires-python = ">=3.9.0" -authors = [ - { name = "dbt Labs", email = "info@dbtlabs.com" }, -] -maintainers = [ - { name = "dbt Labs", email = "info@dbtlabs.com" }, -] +authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] classifiers = [ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", From 2614d447c08a2d5425367eee9966f943fc03075c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 00:25:15 -0500 Subject: [PATCH 14/62] make the release prep call local --- .bumpversion.cfg | 0 .github/workflows/release.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 18fa346e6..e3e86c151 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,7 +121,7 @@ defaults: jobs: release-prep: name: "Release prep: generate changelog, bump version" - uses: dbt-labs/dbt-snowflake/.github/workflows/release_prep_hatch.yml@main + uses: ./.github/workflows/release_prep_hatch.yml with: branch: ${{ inputs.branch }} version: ${{ inputs.version }} From c01a3b909264a9c0dc8721c23229865463bbc3e9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 5 Dec 2024 02:56:58 -0500 Subject: [PATCH 15/62] use hatch to get the current version --- .github/workflows/nightly-release.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 16a5d0da1..17bf8102f 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -50,11 +50,15 @@ jobs: commit_sha=$(git rev-parse HEAD) echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - uses: pypa/hatch@install + + - id: version-number-sources + run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" id: semver From 7000210fc4c6da662cd621f407540043b6b7f7c5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:00:03 -0500 Subject: [PATCH 16/62] minimize changes to main workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da398b9ee..f545c9d62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -105,7 +105,7 @@ jobs: - name: Build distributions run: hatch build - - name: Show distribution contents + - name: Show distributions run: ls -lh dist/ - name: Check distribution descriptions From 07442aae44ce95f273f47bc1e24c07464517e827 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:05:59 -0500 Subject: [PATCH 17/62] merge main into feature branch --- setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index e69de29bb..000000000 From 36e6f859dc910120401b29d12433f24af88f8b3e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:09:41 -0500 Subject: [PATCH 18/62] add the init file in the integration tests root --- tests/functional/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/functional/__init__.py diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 000000000..e69de29bb From 7484c8bd32badd976fbd532f660767f2032b5b45 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:14:20 -0500 Subject: [PATCH 19/62] add the working directory for integration tests --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b9fad5096..7f32cd8fa 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -205,6 +205,7 @@ jobs: - uses: pypa/hatch@install - run: hatch run integration-tests --ddtrace + working-directory: ./dbt-postgres env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 25f29503da55f4f42cabd760e50f6dd2fc42c58b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:15:56 -0500 Subject: [PATCH 20/62] add the working directory for integration tests --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7f32cd8fa..a395a0e79 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -205,7 +205,7 @@ jobs: - uses: pypa/hatch@install - run: hatch run integration-tests --ddtrace - working-directory: ./dbt-postgres + working-directory: ./dbt-snowflake env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 2a0f19404e8c019b28ad59642c254fa645c03574 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:19:04 -0500 Subject: [PATCH 21/62] remove the working directory from integration tests --- .github/workflows/integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a395a0e79..b9fad5096 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -205,7 +205,6 @@ jobs: - uses: pypa/hatch@install - run: hatch run integration-tests --ddtrace - working-directory: ./dbt-snowflake env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From a42d081c7788c022db9a497524245738e9b38453 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:27:32 -0500 Subject: [PATCH 22/62] remove the base init file in tests for pytest --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 5e1ddf35b6eb8c1a74fb947e36f40ff8baac47db Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:45:37 -0500 Subject: [PATCH 23/62] remove ddtrace for debugging --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b9fad5096..026e980ad 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -204,7 +204,7 @@ jobs: - uses: pypa/hatch@install - - run: hatch run integration-tests --ddtrace + - run: hatch run integration-tests env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 12ac39c4ec6adefc39a8ccd9a4fa652632301b60 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:50:29 -0500 Subject: [PATCH 24/62] add ddtrace back, with explicit tests/functional --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 026e980ad..bd51ab727 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -204,7 +204,7 @@ jobs: - uses: pypa/hatch@install - - run: hatch run integration-tests + - run: hatch run integration-tests tests/functional --ddtrace env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 68165c5b5eac4e9c61a59479a6d4bbeb8fbe7f7c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 15:59:56 -0500 Subject: [PATCH 25/62] remove ddtrace again --- .github/workflows/integration.yml | 2 +- hatch.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bd51ab727..f6c70aee6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -83,7 +83,7 @@ jobs: github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.generate-matrix.outputs.result }} diff --git a/hatch.toml b/hatch.toml index 01465cbc0..2e5c56348 100644 --- a/hatch.toml +++ b/hatch.toml @@ -17,7 +17,7 @@ dependencies = [ "ipdb~=0.13.13", "pre-commit==3.7.0", "freezegun", - "pytest", + "pytest>=7.0,<8.0", "pytest-csv~=3.0", "pytest-dotenv", "pytest-logbook~=1.2", From 87365f335487afdaf455277d7b0511d2f1af955d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 16:02:46 -0500 Subject: [PATCH 26/62] remove ddtrace again --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f6c70aee6..bd51ab727 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -83,7 +83,7 @@ jobs: github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: matrix: ${{ steps.generate-matrix.outputs.result }} From 0273da4cc1ce094c4d31ce5650dfd9096a475910 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 9 Dec 2024 12:59:53 -0500 Subject: [PATCH 27/62] hard pin ubuntu images to avoid network disconnects while GHA migrates ubuntu-latest --- .github/scripts/integration-test-matrix.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index e2c88b00b..5ce879169 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -29,7 +29,7 @@ module.exports = ({ context }) => { ) { // always run tests on ubuntu by default include.push({ - os: "ubuntu-latest", + os: "ubuntu-22.04", adapter, "python-version": pythonVersion, }); @@ -68,7 +68,7 @@ module.exports = ({ context }) => { for (const adapter of supportedAdapters) { for (const pythonVersion of supportedPythonVersions) { include.push({ - os: 'ubuntu-latest', + os: 'ubuntu-22.04', adapter: adapter, "python-version": pythonVersion, }); @@ -78,7 +78,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-14"]) { + for (const operatingSystem of ["windows-22.04", "macos-14"]) { include.push({ os: operatingSystem, adapter: adapter, From bd0c4ab9b4876b3d582580c2bbbd3ab76989a52b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 9 Dec 2024 19:07:38 -0500 Subject: [PATCH 28/62] fix dbt dependency updated; remove extra python setup call --- .github/scripts/update_dependencies.sh | 4 ++-- .github/scripts/update_dev_dependency_branches.sh | 2 +- .github/workflows/integration.yml | 7 ++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh index 7880b0163..fabdadff2 100644 --- a/.github/scripts/update_dependencies.sh +++ b/.github/scripts/update_dependencies.sh @@ -3,8 +3,8 @@ set -e git_branch=$1 target_req_file="hatch.toml" -core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" -tests_req_sed_pattern="s|dbt-adapters.git#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" +tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index c78029e5e..9385cf885 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -6,7 +6,7 @@ dbt_adapters_branch=$1 dbt_core_branch=$2 dbt_common_branch=$3 target_req_file="hatch.toml" -core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bd51ab727..3e1bc9a44 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -198,11 +198,8 @@ jobs: ${{ inputs.dbt_common_branch }} cat hatch.toml - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - uses: pypa/hatch@install + - name: Install python dependencies + uses: pypa/hatch@install - run: hatch run integration-tests tests/functional --ddtrace env: From c3faabc5ccc92e6705b7abc0080a745e38b0afed Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 16:07:07 -0500 Subject: [PATCH 29/62] revert os pin --- .github/scripts/integration-test-matrix.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 5ce879169..e2c88b00b 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -29,7 +29,7 @@ module.exports = ({ context }) => { ) { // always run tests on ubuntu by default include.push({ - os: "ubuntu-22.04", + os: "ubuntu-latest", adapter, "python-version": pythonVersion, }); @@ -68,7 +68,7 @@ module.exports = ({ context }) => { for (const adapter of supportedAdapters) { for (const pythonVersion of supportedPythonVersions) { include.push({ - os: 'ubuntu-22.04', + os: 'ubuntu-latest', adapter: adapter, "python-version": pythonVersion, }); @@ -78,7 +78,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-22.04", "macos-14"]) { + for (const operatingSystem of ["windows-latest", "macos-14"]) { include.push({ os: operatingSystem, adapter: adapter, From 596876e9f60f643337582bcbe58f2fb07a632e36 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 16:07:21 -0500 Subject: [PATCH 30/62] add dbt-tests-adapter to the dev deps update --- .github/scripts/update_dev_dependency_branches.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index 9385cf885..603dec17b 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -8,14 +8,17 @@ dbt_common_branch=$3 target_req_file="hatch.toml" core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${dbt_adapters_branch}#subdirectory=dbt-tests-adapter|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$tests_req_sed_pattern" $target_req_file sed -i "" "$core_req_sed_pattern" $target_req_file sed -i "" "$common_req_sed_pattern" $target_req_file else sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$tests_req_sed_pattern" $target_req_file sed -i "$core_req_sed_pattern" $target_req_file sed -i "$common_req_sed_pattern" $target_req_file fi From 5aaa7c4239d149d6a5b4b3e42e545427fa33a511 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 16:54:46 -0500 Subject: [PATCH 31/62] add tests root init file back --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb From 8e9ef3a29afbf23805354720ae4e7c4bb69b379f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:10:29 -0500 Subject: [PATCH 32/62] move test matrix into the workflow; remove call to ddtrace --- .github/scripts/integration-test-matrix.js | 95 ---------------------- .github/workflows/integration.yml | 71 ++-------------- 2 files changed, 9 insertions(+), 157 deletions(-) delete mode 100644 .github/scripts/integration-test-matrix.js diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index e2c88b00b..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,95 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.9"; - const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"]; - const supportedAdapters = ["snowflake"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const changes = JSON.parse(process.env.CHANGES); - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - console.log("changes", changes); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - if ( - changes.includes(adapter) || - testAllLabel || - labels.includes(`test ${adapter}`) - ) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-14", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-14"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3e1bc9a44..d27695b8f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -76,82 +76,29 @@ defaults: shell: bash jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v3 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - snowflake: - - '.github/**/*.yml' - - '.github/**/*.sh' - - 'dbt/**' - - 'tests/**' - - 'hatch.toml' - - 'pyproject.toml' - - '*.py' - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v7 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix test: name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} # run if not a PR from a forked repository or has a label to mark as safe to test # also checks that the matrix generated is not empty if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && ( github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ok to test') ) runs-on: ${{ matrix.os }} - needs: test-metadata strategy: fail-fast: false - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} + matrix: + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + include: + - os: macos-14 + python-version: "3.9" + - os: windows-2022 + python-version: "3.9" env: DBT_INVOCATION_ENV: github-actions @@ -201,7 +148,7 @@ jobs: - name: Install python dependencies uses: pypa/hatch@install - - run: hatch run integration-tests tests/functional --ddtrace + - run: hatch run integration-tests env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 85d792ae0262bb6a39a453d8c3727c1f2f23b235 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:28:41 -0500 Subject: [PATCH 33/62] fix pytest config in pyproject.toml --- pyproject.toml | 3 ++- .env.example => test.env.example | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename .env.example => test.env.example (100%) diff --git a/pyproject.toml b/pyproject.toml index 9cb49cc93..3d6c6b24a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,9 @@ Repository = "https://github.com/dbt-labs/dbt-snowflake.git" Issues = "https://github.com/dbt-labs/dbt-snowflake/issues" Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" -[tool.pytest] +[tool.pytest.ini_options] testpaths = ["tests/functional", "tests/unit"] +env_files = ["test.env"] addopts = "-v -n auto" color = true filterwarnings = [ diff --git a/.env.example b/test.env.example similarity index 100% rename from .env.example rename to test.env.example From d67dd33af00b45b78a4cc735ad78de33fa09d717 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:33:29 -0500 Subject: [PATCH 34/62] fix job name --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d27695b8f..e940c92f0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -77,7 +77,7 @@ defaults: jobs: test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} + name: ${{ matrix.os }} / python ${{ matrix.python-version }} # run if not a PR from a forked repository or has a label to mark as safe to test # also checks that the matrix generated is not empty From aa1b2c35bc397c2165212e7f1cb5cee0dc92e41f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:35:33 -0500 Subject: [PATCH 35/62] remove unused script --- .github/scripts/update_dependencies.sh | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .github/scripts/update_dependencies.sh diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh deleted file mode 100644 index fabdadff2..000000000 --- a/.github/scripts/update_dependencies.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="hatch.toml" -core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" -tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi From 8670b23339c08a253f1c4fa1845e4d9f994ba91d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 17:39:06 -0500 Subject: [PATCH 36/62] reduce pr noise --- tests/functional/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/functional/__init__.py diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 58eb973ba50037217f6cfe3d74523bc05e59346c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 20:45:23 -0500 Subject: [PATCH 37/62] fix dependency pins --- hatch.toml | 10 ++++------ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/hatch.toml b/hatch.toml index 2e5c56348..91aa779dc 100644 --- a/hatch.toml +++ b/hatch.toml @@ -15,14 +15,12 @@ dependencies = [ "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", "ddtrace==2.3.0", "ipdb~=0.13.13", - "pre-commit==3.7.0", - "freezegun", - "pytest>=7.0,<8.0", + "pre-commit~=3.7.0", + "pytest~=7.4", "pytest-csv~=3.0", - "pytest-dotenv", + "pytest-dotenv~=0.5.2", "pytest-logbook~=1.2", - "pytest-mock", - "pytest-xdist", + "pytest-xdist~=3.6", ] [envs.default.scripts] diff --git a/pyproject.toml b/pyproject.toml index 3d6c6b24a..99812e929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters>=1.10.4,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From ab5eeeffb52fcbf3e8e35f41260df75ea6568411 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 21:14:38 -0500 Subject: [PATCH 38/62] fix pytest config --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 99812e929..2a988d5ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,7 @@ Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" [tool.pytest.ini_options] testpaths = ["tests/functional", "tests/unit"] env_files = ["test.env"] -addopts = "-v -n auto" -color = true +addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", From acd933e40d4434cea07f77c19ab8ea29c6ba8ed1 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 10 Dec 2024 22:04:49 -0500 Subject: [PATCH 39/62] pin snowflake sdk to troubleshoot ci memory issues --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2a988d5ec..f8ff534f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ dependencies = [ "dbt-common>=1.10,<2.0", "dbt-adapters>=1.10.4,<2.0", - "snowflake-connector-python[secure-local-storage]~=3.0", + "snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core From 7c2ae97b585e8f6352e0edafe03cab85db3dfec0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 10:46:46 -0500 Subject: [PATCH 40/62] add ddtrace back; it doesn't look like that's causing the memory issue --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e940c92f0..ab23a7de8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -148,7 +148,7 @@ jobs: - name: Install python dependencies uses: pypa/hatch@install - - run: hatch run integration-tests + - run: hatch run integration-tests --ddtrace env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} From 3d64a461e685e8565acf16ac1b82254a65df375b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 10:47:25 -0500 Subject: [PATCH 41/62] break out incremental tests from other tests because that's where CI is stopping with memory issues in GHA on Ubuntu --- hatch.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hatch.toml b/hatch.toml index 91aa779dc..15581229d 100644 --- a/hatch.toml +++ b/hatch.toml @@ -27,10 +27,13 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" -integration-tests = "python -m pytest {args:tests/functional}" +integration-tests = [ + "- python -m pytest {args:tests/functional} --ignore=tests/functional/adapter/incremental", + "- python -m pytest tests/functional/adapter/incremental {args}", +] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", - "docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev", + "docker run --rm -it --name dbt-snowflake-dev -v $(pwd):/opt/code dbt-snowflake-dev", ] [envs.build] From ac29ea2f62203d7e81ff990bf013650e90b0b32a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 11:16:24 -0500 Subject: [PATCH 42/62] break out tests into one module at a time to troubleshoot memory issues in GHA on Ubuntu --- hatch.toml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/hatch.toml b/hatch.toml index 15581229d..dc218aed6 100644 --- a/hatch.toml +++ b/hatch.toml @@ -28,8 +28,36 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ - "- python -m pytest {args:tests/functional} --ignore=tests/functional/adapter/incremental", + "- python -m pytest tests/functional/adapter/catalog_tests {args}", + "- python -m pytest tests/functional/adapter/column_types {args}", + "- python -m pytest tests/functional/adapter/custom_schema_tests {args}", + "- python -m pytest tests/functional/adapter/dbt_clone {args}", + "- python -m pytest tests/functional/adapter/dbt_show {args}", + "- python -m pytest tests/functional/adapter/empty {args}", "- python -m pytest tests/functional/adapter/incremental {args}", + "- python -m pytest tests/functional/adapter/list_relations_tests {args}", + "- python -m pytest tests/functional/adapter/python_model_tests {args}", + "- python -m pytest tests/functional/adapter/query_comment_tests {args}", + "- python -m pytest tests/functional/adapter/simple_copy {args}", + "- python -m pytest tests/functional/adapter/simple_seed {args}", + "- python -m pytest tests/functional/adapter/statement_test {args}", + "- python -m pytest tests/functional/adapter/unit_testing {args}", + "- python -m pytest tests/functional/adapter/utils {args}", + "- python -m pytest tests/functional/adapter/test_aliases.py {args}", + "- python -m pytest tests/functional/adapter/test_anonymous_usage_stats.py {args}", + "- python -m pytest tests/functional/adapter/test_basic.py {args}", + "- python -m pytest tests/functional/adapter/test_caching.py {args}", + "- python -m pytest tests/functional/adapter/test_changing_relation_type.py {args}", + "- python -m pytest tests/functional/adapter/test_concurrency.py {args}", + "- python -m pytest tests/functional/adapter/test_constraints.py {args}", + "- python -m pytest tests/functional/adapter/test_emphemeral.py {args}", + "- python -m pytest tests/functional/adapter/test_get_last_relation_modified.py {args}", + "- python -m pytest tests/functional/adapter/test_grants.py {args}", + "- python -m pytest tests/functional/adapter/test_incremental_microbatch.py {args}", + "- python -m pytest tests/functional/adapter/test_persist_docs.py {args}", + "- python -m pytest tests/functional/adapter/test_python_model.py {args}", + "- python -m pytest tests/functional/adapter/test_simple_snapshot.py {args}", + "- python -m pytest tests/functional/adapter/test_timestamps.py {args}", ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", From 8f0d68c77f02ae97c0256c295a47e8da73eb0acc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 12:05:28 -0500 Subject: [PATCH 43/62] add filter for utcnow deprecation, remove filter for resource warning --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8ff534f3..88869051b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,5 +45,5 @@ env_files = ["test.env"] addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", - "ignore:unclosed file .*:ResourceWarning", + "ignore:*DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version.*", ] From 78469fe17a9af3dd1ef594bac2a4cb5f7252dbf3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 12:10:55 -0500 Subject: [PATCH 44/62] add filter for utcnow deprecation, remove filter for resource warning --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 88869051b..7be51a5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,5 +45,5 @@ env_files = ["test.env"] addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", - "ignore:*DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version.*", + "ignore:*datetime.datetime.utcnow() is deprecated*:DeprecationWarning", ] From 7f72f5625aad9e669e4e48c63ff84aa645e1c21e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 12:13:51 -0500 Subject: [PATCH 45/62] add filter for utcnow deprecation, remove filter for resource warning --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7be51a5e9..8d89f31fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,5 +45,5 @@ env_files = ["test.env"] addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", - "ignore:*datetime.datetime.utcnow() is deprecated*:DeprecationWarning", + "ignore:.*datetime.datetime.utcnow() is deprecated*:DeprecationWarning", ] From 5e76a7e3f1d97030b1853b621c110682e49bfdf2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 13:00:10 -0500 Subject: [PATCH 46/62] add filter for utcnow deprecation, remove other filters --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8d89f31fa..fac440a73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,5 @@ testpaths = ["tests/functional", "tests/unit"] env_files = ["test.env"] addopts = "-v --color=yes -n auto" filterwarnings = [ - "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", - "ignore:.*datetime.datetime.utcnow() is deprecated*:DeprecationWarning", + "ignore:datetime.datetime.utcnow() is deprecated*:DeprecationWarning", ] From c82b6b6b03d5820eda7fd046be9ce30759202b34 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 13:47:46 -0500 Subject: [PATCH 47/62] remove pytest warning filters --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fac440a73..28ecbc415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,3 @@ Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" testpaths = ["tests/functional", "tests/unit"] env_files = ["test.env"] addopts = "-v --color=yes -n auto" -filterwarnings = [ - "ignore:datetime.datetime.utcnow() is deprecated*:DeprecationWarning", -] From 57559402ffffab35f3ef88e757c08fbbd14d6791 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 14:01:17 -0500 Subject: [PATCH 48/62] remove pytest warning filter for utcnow deprecation --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 28ecbc415..568aa3533 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,3 +43,6 @@ Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" testpaths = ["tests/functional", "tests/unit"] env_files = ["test.env"] addopts = "-v --color=yes -n auto" +filterwarnings = [ + "ignore:datetime.datetime.utcnow:DeprecationWarning", +] From 9377c5f856fb163823214c48b6a98b639d9747c4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 15:50:49 -0500 Subject: [PATCH 49/62] add test groups to troubleshoot where the memory issue on ubuntu --- hatch.toml | 36 ++++------------------ pyproject.toml | 8 +++++ tests/functional/conftest.py | 60 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 tests/functional/conftest.py diff --git a/hatch.toml b/hatch.toml index dc218aed6..5ad6c4075 100644 --- a/hatch.toml +++ b/hatch.toml @@ -28,36 +28,12 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ - "- python -m pytest tests/functional/adapter/catalog_tests {args}", - "- python -m pytest tests/functional/adapter/column_types {args}", - "- python -m pytest tests/functional/adapter/custom_schema_tests {args}", - "- python -m pytest tests/functional/adapter/dbt_clone {args}", - "- python -m pytest tests/functional/adapter/dbt_show {args}", - "- python -m pytest tests/functional/adapter/empty {args}", - "- python -m pytest tests/functional/adapter/incremental {args}", - "- python -m pytest tests/functional/adapter/list_relations_tests {args}", - "- python -m pytest tests/functional/adapter/python_model_tests {args}", - "- python -m pytest tests/functional/adapter/query_comment_tests {args}", - "- python -m pytest tests/functional/adapter/simple_copy {args}", - "- python -m pytest tests/functional/adapter/simple_seed {args}", - "- python -m pytest tests/functional/adapter/statement_test {args}", - "- python -m pytest tests/functional/adapter/unit_testing {args}", - "- python -m pytest tests/functional/adapter/utils {args}", - "- python -m pytest tests/functional/adapter/test_aliases.py {args}", - "- python -m pytest tests/functional/adapter/test_anonymous_usage_stats.py {args}", - "- python -m pytest tests/functional/adapter/test_basic.py {args}", - "- python -m pytest tests/functional/adapter/test_caching.py {args}", - "- python -m pytest tests/functional/adapter/test_changing_relation_type.py {args}", - "- python -m pytest tests/functional/adapter/test_concurrency.py {args}", - "- python -m pytest tests/functional/adapter/test_constraints.py {args}", - "- python -m pytest tests/functional/adapter/test_emphemeral.py {args}", - "- python -m pytest tests/functional/adapter/test_get_last_relation_modified.py {args}", - "- python -m pytest tests/functional/adapter/test_grants.py {args}", - "- python -m pytest tests/functional/adapter/test_incremental_microbatch.py {args}", - "- python -m pytest tests/functional/adapter/test_persist_docs.py {args}", - "- python -m pytest tests/functional/adapter/test_python_model.py {args}", - "- python -m pytest tests/functional/adapter/test_simple_snapshot.py {args}", - "- python -m pytest tests/functional/adapter/test_timestamps.py {args}", + '- python -m pytest -m "group_1" {args:tests/functional}', + '- python -m pytest -m "group_2" {args:tests/functional}', + '- python -m pytest -m "group_3" {args:tests/functional}', + '- python -m pytest -m "group_4" {args:tests/functional}', + '- python -m pytest -m "group_5" {args:tests/functional}', + '- python -m pytest -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5" {args:tests/functional}', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index 568aa3533..4f1e2b9c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,11 @@ addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] +markers = [ + "group_1", + "group_2", + "group_3", + "group_4", + "group_5", + "group_6", +] diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 000000000..70cb2901c --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,60 @@ +import pathlib + +import pytest + + +TEST_GROUPS = { + "catalog_tests": "group_1", + "column_types": "group_1", + "custom_schema_tests": "group_1", + "dbt_clone": "group_1", + "dbt_show": "group_1", + "empty": "group_1", + "incremental": "group_2", + "list_relations_tests": "group_2", + "python_model_tests": "group_2", + "query_comment_tests": "group_2", + "simple_copy": "group_2", + "simple_seed": "group_2", + "statement_test": "group_3", + "store_test_failures_tests": "group_3", + "unit_testing": "group_3", + "utils": "group_3", + "test_aliases.py": "group_3", + "test_anonymous_usage_stats.py": "group_3", + "test_basic.py": "group_3", + "test_caching.py": "group_4", + "test_changing_relation_type.py": "group_4", + "test_concurrency.py": "group_4", + "test_constraints.py": "group_4", + "test_ephemeral.py": "group_4", + "test_get_last_relation_modified.py": "group_4", + "test_grants.py": "group_5", + "test_incremental_microbatch.py": "group_5", + "test_persist_docs.py": "group_5", + "test_python_model.py": "group_5", + "test_simple_snapshot.py": "group_5", + "test_timestamps.py": "group_5", + "auth_tests": "group_6", + "generic_test_tests": "group_6", + "iceberg": "group_6", + "override_database": "group_6", + "query_tag": "group_6", + "redact_log_values": "group_6", + "relation_tests": "group_6", + "snowflake_view_dependency": "group_6", + "warehouse_test": "group_6", + "test_isolated_begin_commit.py": "group_6", +} + + +def pytest_collection_modifyitems(config, items): + test_root = pathlib.Path(config.rootdir) / "tests" / "functional" + for item in items: + test_path = pathlib.Path(item.fspath).relative_to(test_root) + test_module = test_path.parts[0] + if test_module == "adapter": + test_module = test_path.parts[1] + if mark_name := TEST_GROUPS.get(test_module): + mark = getattr(pytest.mark, mark_name) + item.add_marker(mark) From c5248224a121052e6da3a04c8ba6c7a10b474bae Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 16:24:22 -0500 Subject: [PATCH 50/62] swap args order for pytest, don't mark anything outside of functional tests --- hatch.toml | 12 ++++++------ tests/functional/conftest.py | 5 ++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/hatch.toml b/hatch.toml index 5ad6c4075..cb96c71d4 100644 --- a/hatch.toml +++ b/hatch.toml @@ -28,12 +28,12 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ - '- python -m pytest -m "group_1" {args:tests/functional}', - '- python -m pytest -m "group_2" {args:tests/functional}', - '- python -m pytest -m "group_3" {args:tests/functional}', - '- python -m pytest -m "group_4" {args:tests/functional}', - '- python -m pytest -m "group_5" {args:tests/functional}', - '- python -m pytest -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5" {args:tests/functional}', + '- python -m pytest {args:tests/functional} -m "group_1"', + '- python -m pytest {args:tests/functional} -m "group_2"', + '- python -m pytest {args:tests/functional} -m "group_3"', + '- python -m pytest {args:tests/functional} -m "group_4"', + '- python -m pytest {args:tests/functional} -m "group_5"', + '- python -m pytest {args:tests/functional} -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 70cb2901c..025164da3 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -51,7 +51,10 @@ def pytest_collection_modifyitems(config, items): test_root = pathlib.Path(config.rootdir) / "tests" / "functional" for item in items: - test_path = pathlib.Path(item.fspath).relative_to(test_root) + try: + test_path = pathlib.Path(item.fspath).relative_to(test_root) + except ValueError: + continue test_module = test_path.parts[0] if test_module == "adapter": test_module = test_path.parts[1] From 58d8b66524f638d09311b880149374ce734dd22f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 16:56:00 -0500 Subject: [PATCH 51/62] group_6 was missing --- hatch.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hatch.toml b/hatch.toml index cb96c71d4..3b58ef6d1 100644 --- a/hatch.toml +++ b/hatch.toml @@ -33,7 +33,8 @@ integration-tests = [ '- python -m pytest {args:tests/functional} -m "group_3"', '- python -m pytest {args:tests/functional} -m "group_4"', '- python -m pytest {args:tests/functional} -m "group_5"', - '- python -m pytest {args:tests/functional} -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5"', + '- python -m pytest {args:tests/functional} -m "group_6"', + '- python -m pytest {args:tests/functional} -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5 and not group_6"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", From b069393a306dadff2b6f2d2e5f4b8a000ee77a17 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 17:36:12 -0500 Subject: [PATCH 52/62] remove group_2 and isolate newer tests into group_6 --- hatch.toml | 3 +-- pyproject.toml | 1 - tests/functional/conftest.py | 28 ++++++++++++++-------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/hatch.toml b/hatch.toml index 3b58ef6d1..fafac5a40 100644 --- a/hatch.toml +++ b/hatch.toml @@ -29,12 +29,11 @@ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ '- python -m pytest {args:tests/functional} -m "group_1"', - '- python -m pytest {args:tests/functional} -m "group_2"', '- python -m pytest {args:tests/functional} -m "group_3"', '- python -m pytest {args:tests/functional} -m "group_4"', '- python -m pytest {args:tests/functional} -m "group_5"', '- python -m pytest {args:tests/functional} -m "group_6"', - '- python -m pytest {args:tests/functional} -m "not group_1 and not group_2 and not group_3 and not group_4 and not group_5 and not group_6"', + '- python -m pytest {args:tests/functional} -m "not group_1 and not group_3 and not group_4 and not group_5 and not group_6"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index 4f1e2b9c3..4f789af06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ filterwarnings = [ ] markers = [ "group_1", - "group_2", "group_3", "group_4", "group_5", diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 025164da3..a94aa376f 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -10,12 +10,12 @@ "dbt_clone": "group_1", "dbt_show": "group_1", "empty": "group_1", - "incremental": "group_2", - "list_relations_tests": "group_2", - "python_model_tests": "group_2", - "query_comment_tests": "group_2", - "simple_copy": "group_2", - "simple_seed": "group_2", + "incremental": "group_6", + "list_relations_tests": "group_6", + "python_model_tests": "group_1", + "query_comment_tests": "group_1", + "simple_copy": "group_1", + "simple_seed": "group_1", "statement_test": "group_3", "store_test_failures_tests": "group_3", "unit_testing": "group_3", @@ -35,16 +35,16 @@ "test_python_model.py": "group_5", "test_simple_snapshot.py": "group_5", "test_timestamps.py": "group_5", - "auth_tests": "group_6", - "generic_test_tests": "group_6", - "iceberg": "group_6", - "override_database": "group_6", - "query_tag": "group_6", - "redact_log_values": "group_6", + "auth_tests": "group_1", + "generic_test_tests": "group_1", + "iceberg": "group_1", + "override_database": "group_1", + "query_tag": "group_1", + "redact_log_values": "group_1", "relation_tests": "group_6", "snowflake_view_dependency": "group_6", - "warehouse_test": "group_6", - "test_isolated_begin_commit.py": "group_6", + "warehouse_test": "group_1", + "test_isolated_begin_commit.py": "group_1", } From 3439bc44dae3f60156ec830ae367db7f9aa99898 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 18:05:31 -0500 Subject: [PATCH 53/62] remove group_3 and group_4 --- hatch.toml | 3 --- pyproject.toml | 2 -- tests/functional/conftest.py | 26 +++++++++++++------------- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/hatch.toml b/hatch.toml index fafac5a40..b96d7622e 100644 --- a/hatch.toml +++ b/hatch.toml @@ -29,11 +29,8 @@ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ '- python -m pytest {args:tests/functional} -m "group_1"', - '- python -m pytest {args:tests/functional} -m "group_3"', - '- python -m pytest {args:tests/functional} -m "group_4"', '- python -m pytest {args:tests/functional} -m "group_5"', '- python -m pytest {args:tests/functional} -m "group_6"', - '- python -m pytest {args:tests/functional} -m "not group_1 and not group_3 and not group_4 and not group_5 and not group_6"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index 4f789af06..8fa4bcec7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,8 +48,6 @@ filterwarnings = [ ] markers = [ "group_1", - "group_3", - "group_4", "group_5", "group_6", ] diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index a94aa376f..52e44b5ed 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -16,19 +16,19 @@ "query_comment_tests": "group_1", "simple_copy": "group_1", "simple_seed": "group_1", - "statement_test": "group_3", - "store_test_failures_tests": "group_3", - "unit_testing": "group_3", - "utils": "group_3", - "test_aliases.py": "group_3", - "test_anonymous_usage_stats.py": "group_3", - "test_basic.py": "group_3", - "test_caching.py": "group_4", - "test_changing_relation_type.py": "group_4", - "test_concurrency.py": "group_4", - "test_constraints.py": "group_4", - "test_ephemeral.py": "group_4", - "test_get_last_relation_modified.py": "group_4", + "statement_test": "group_1", + "store_test_failures_tests": "group_1", + "unit_testing": "group_1", + "utils": "group_1", + "test_aliases.py": "group_1", + "test_anonymous_usage_stats.py": "group_1", + "test_basic.py": "group_1", + "test_caching.py": "group_1", + "test_changing_relation_type.py": "group_1", + "test_concurrency.py": "group_1", + "test_constraints.py": "group_1", + "test_ephemeral.py": "group_1", + "test_get_last_relation_modified.py": "group_1", "test_grants.py": "group_5", "test_incremental_microbatch.py": "group_5", "test_persist_docs.py": "group_5", From b85682a6079ddd93d79caf2c8067673f358a35ee Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 18:44:24 -0500 Subject: [PATCH 54/62] add group_4 back in --- hatch.toml | 1 + pyproject.toml | 1 + tests/functional/conftest.py | 12 ++++++------ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hatch.toml b/hatch.toml index b96d7622e..e5da3ecbd 100644 --- a/hatch.toml +++ b/hatch.toml @@ -29,6 +29,7 @@ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ '- python -m pytest {args:tests/functional} -m "group_1"', + '- python -m pytest {args:tests/functional} -m "group_4"', '- python -m pytest {args:tests/functional} -m "group_5"', '- python -m pytest {args:tests/functional} -m "group_6"', ] diff --git a/pyproject.toml b/pyproject.toml index 8fa4bcec7..3620301d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ filterwarnings = [ ] markers = [ "group_1", + "group_4", "group_5", "group_6", ] diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 52e44b5ed..46c9ca10b 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -23,12 +23,12 @@ "test_aliases.py": "group_1", "test_anonymous_usage_stats.py": "group_1", "test_basic.py": "group_1", - "test_caching.py": "group_1", - "test_changing_relation_type.py": "group_1", - "test_concurrency.py": "group_1", - "test_constraints.py": "group_1", - "test_ephemeral.py": "group_1", - "test_get_last_relation_modified.py": "group_1", + "test_caching.py": "group_4", + "test_changing_relation_type.py": "group_4", + "test_concurrency.py": "group_4", + "test_constraints.py": "group_4", + "test_ephemeral.py": "group_4", + "test_get_last_relation_modified.py": "group_4", "test_grants.py": "group_5", "test_incremental_microbatch.py": "group_5", "test_persist_docs.py": "group_5", From 2b66d211e7fde3d587c54a9e32b44f2592d76b8a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Dec 2024 21:09:21 -0500 Subject: [PATCH 55/62] move more things into group_1 --- tests/functional/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 46c9ca10b..e206d4543 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -29,12 +29,12 @@ "test_constraints.py": "group_4", "test_ephemeral.py": "group_4", "test_get_last_relation_modified.py": "group_4", - "test_grants.py": "group_5", + "test_grants.py": "group_1", "test_incremental_microbatch.py": "group_5", - "test_persist_docs.py": "group_5", + "test_persist_docs.py": "group_1", "test_python_model.py": "group_5", "test_simple_snapshot.py": "group_5", - "test_timestamps.py": "group_5", + "test_timestamps.py": "group_1", "auth_tests": "group_1", "generic_test_tests": "group_1", "iceberg": "group_1", @@ -42,7 +42,7 @@ "query_tag": "group_1", "redact_log_values": "group_1", "relation_tests": "group_6", - "snowflake_view_dependency": "group_6", + "snowflake_view_dependency": "group_1", "warehouse_test": "group_1", "test_isolated_begin_commit.py": "group_1", } From 24c57af48d3ea6ea0fa81f47d8d26f971339dc19 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 16 Dec 2024 20:09:16 -0500 Subject: [PATCH 56/62] group tests by when they were updated for troubleshooting --- hatch.toml | 8 ++-- pyproject.toml | 8 ++-- tests/functional/conftest.py | 82 ++++++++++++++++++------------------ 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/hatch.toml b/hatch.toml index e5da3ecbd..c3d3847ae 100644 --- a/hatch.toml +++ b/hatch.toml @@ -28,10 +28,10 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ - '- python -m pytest {args:tests/functional} -m "group_1"', - '- python -m pytest {args:tests/functional} -m "group_4"', - '- python -m pytest {args:tests/functional} -m "group_5"', - '- python -m pytest {args:tests/functional} -m "group_6"', + '- python -m pytest {args:tests/functional} -m "last_year"', + '- python -m pytest {args:tests/functional} -m "last_quarter"', + '- python -m pytest {args:tests/functional} -m "last_month"', + '- python -m pytest {args:tests/functional} -m "microbatch"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index 3620301d9..ab5251059 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] markers = [ - "group_1", - "group_4", - "group_5", - "group_6", + "last_year", + "last_quarter", + "last_month", + "microbatch", ] diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index e206d4543..f1cbc53e4 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -4,47 +4,47 @@ TEST_GROUPS = { - "catalog_tests": "group_1", - "column_types": "group_1", - "custom_schema_tests": "group_1", - "dbt_clone": "group_1", - "dbt_show": "group_1", - "empty": "group_1", - "incremental": "group_6", - "list_relations_tests": "group_6", - "python_model_tests": "group_1", - "query_comment_tests": "group_1", - "simple_copy": "group_1", - "simple_seed": "group_1", - "statement_test": "group_1", - "store_test_failures_tests": "group_1", - "unit_testing": "group_1", - "utils": "group_1", - "test_aliases.py": "group_1", - "test_anonymous_usage_stats.py": "group_1", - "test_basic.py": "group_1", - "test_caching.py": "group_4", - "test_changing_relation_type.py": "group_4", - "test_concurrency.py": "group_4", - "test_constraints.py": "group_4", - "test_ephemeral.py": "group_4", - "test_get_last_relation_modified.py": "group_4", - "test_grants.py": "group_1", - "test_incremental_microbatch.py": "group_5", - "test_persist_docs.py": "group_1", - "test_python_model.py": "group_5", - "test_simple_snapshot.py": "group_5", - "test_timestamps.py": "group_1", - "auth_tests": "group_1", - "generic_test_tests": "group_1", - "iceberg": "group_1", - "override_database": "group_1", - "query_tag": "group_1", - "redact_log_values": "group_1", - "relation_tests": "group_6", - "snowflake_view_dependency": "group_1", - "warehouse_test": "group_1", - "test_isolated_begin_commit.py": "group_1", + "catalog_tests": "last_year", + "column_types": "last_year", + "custom_schema_tests": "last_year", + "dbt_clone": "last_quarter", + "dbt_show": "last_quarter", + "empty": "last_quarter", + "incremental": "microbatch", + "list_relations_tests": "last_month", + "python_model_tests": "last_year", + "query_comment_tests": "last_year", + "simple_copy": "last_year", + "simple_seed": "last_year", + "statement_test": "last_year", + "store_test_failures_tests": "last_year", + "unit_testing": "last_year", + "utils": "last_year", + "test_aliases.py": "last_year", + "test_anonymous_usage_stats.py": "last_year", + "test_basic.py": "last_year", + "test_caching.py": "last_year", + "test_changing_relation_type.py": "last_year", + "test_concurrency.py": "last_year", + "test_constraints.py": "last_year", + "test_ephemeral.py": "last_year", + "test_get_last_relation_modified.py": "last_year", + "test_grants.py": "last_year", + "test_incremental_microbatch.py": "microbatch", + "test_persist_docs.py": "last_year", + "test_python_model.py": "last_year", + "test_simple_snapshot.py": "last_year", + "test_timestamps.py": "last_year", + "auth_tests": "last_month", + "generic_test_tests": "last_month", + "iceberg": "last_month", + "override_database": "last_year", + "query_tag": "last_month", + "redact_log_values": "last_year", + "relation_tests": "last_month", + "snowflake_view_dependency": "last_year", + "warehouse_test": "last_quarter", + "test_isolated_begin_commit.py": "last_year", } From 764dbdd9eb3c5d16a071b52fc3a4147c5af1ff87 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 17 Dec 2024 14:35:31 -0500 Subject: [PATCH 57/62] pull out python model tests for troubleshooting --- hatch.toml | 2 ++ pyproject.toml | 2 ++ tests/functional/conftest.py | 6 +++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hatch.toml b/hatch.toml index c3d3847ae..5543df3a1 100644 --- a/hatch.toml +++ b/hatch.toml @@ -32,6 +32,8 @@ integration-tests = [ '- python -m pytest {args:tests/functional} -m "last_quarter"', '- python -m pytest {args:tests/functional} -m "last_month"', '- python -m pytest {args:tests/functional} -m "microbatch"', + '- python -m pytest {args:tests/functional} -m "python_model"', + '- python -m pytest {args:tests/functional} -m "large"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index ab5251059..cb24785a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,4 +51,6 @@ markers = [ "last_quarter", "last_month", "microbatch", + "python_model", + "large", ] diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index f1cbc53e4..eb1a992bd 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -12,7 +12,7 @@ "empty": "last_quarter", "incremental": "microbatch", "list_relations_tests": "last_month", - "python_model_tests": "last_year", + "python_model_tests": "python_model", "query_comment_tests": "last_year", "simple_copy": "last_year", "simple_seed": "last_year", @@ -24,7 +24,7 @@ "test_anonymous_usage_stats.py": "last_year", "test_basic.py": "last_year", "test_caching.py": "last_year", - "test_changing_relation_type.py": "last_year", + "test_changing_relation_type.py": "large", "test_concurrency.py": "last_year", "test_constraints.py": "last_year", "test_ephemeral.py": "last_year", @@ -32,7 +32,7 @@ "test_grants.py": "last_year", "test_incremental_microbatch.py": "microbatch", "test_persist_docs.py": "last_year", - "test_python_model.py": "last_year", + "test_python_model.py": "python_model", "test_simple_snapshot.py": "last_year", "test_timestamps.py": "last_year", "auth_tests": "last_month", From 37d2dae93cde8f5f370232a88c871821e2ea45bd Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 17 Dec 2024 19:34:12 -0500 Subject: [PATCH 58/62] mark high memory tests and run as a separate pytest run --- .github/workflows/integration.yml | 2 +- hatch.toml | 9 +-- pyproject.toml | 9 +-- .../adapter/dbt_clone/test_dbt_clone.py | 3 + tests/functional/adapter/test_constraints.py | 4 ++ tests/functional/adapter/test_grants.py | 3 + .../adapter/unit_testing/test_unit_testing.py | 3 + tests/functional/conftest.py | 63 ------------------- 8 files changed, 18 insertions(+), 78 deletions(-) delete mode 100644 tests/functional/conftest.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ab23a7de8..b939e2581 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -148,7 +148,7 @@ jobs: - name: Install python dependencies uses: pypa/hatch@install - - run: hatch run integration-tests --ddtrace + - run: hatch run integration-tests tests/functional --ddtrace env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} diff --git a/hatch.toml b/hatch.toml index 5543df3a1..35410dc3f 100644 --- a/hatch.toml +++ b/hatch.toml @@ -8,6 +8,7 @@ packages = ["dbt"] packages = ["dbt"] [envs.default] +python = "3.9" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", @@ -28,12 +29,8 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = [ - '- python -m pytest {args:tests/functional} -m "last_year"', - '- python -m pytest {args:tests/functional} -m "last_quarter"', - '- python -m pytest {args:tests/functional} -m "last_month"', - '- python -m pytest {args:tests/functional} -m "microbatch"', - '- python -m pytest {args:tests/functional} -m "python_model"', - '- python -m pytest {args:tests/functional} -m "large"', + '- python -m pytest {args:tests/functional} -m "not high_memory"', + '- python -m pytest {args:tests/functional} -m "high_memory"', ] docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", diff --git a/pyproject.toml b/pyproject.toml index cb24785a4..faab5c084 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,11 +46,4 @@ addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] -markers = [ - "last_year", - "last_quarter", - "last_month", - "microbatch", - "python_model", - "large", -] +markers = ["high_memory"] diff --git a/tests/functional/adapter/dbt_clone/test_dbt_clone.py b/tests/functional/adapter/dbt_clone/test_dbt_clone.py index 2a73eb7f4..b7911ac76 100644 --- a/tests/functional/adapter/dbt_clone/test_dbt_clone.py +++ b/tests/functional/adapter/dbt_clone/test_dbt_clone.py @@ -6,6 +6,9 @@ from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseClonePossible +pytestmark = pytest.mark.high_memory + + class TestSnowflakeClonePossible(BaseClonePossible): @pytest.fixture(autouse=True) def clean_up(self, project): diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 03adc3bed..2ba7c2667 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -18,6 +18,10 @@ model_contract_header_schema_yml, ) + +pytestmark = pytest.mark.high_memory + + my_model_contract_sql_header_sql = """ {{ config( diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 30e687f59..9cfde9990 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -6,6 +6,9 @@ from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants +pytestmark = pytest.mark.high_memory + + class BaseCopyGrantsSnowflake: # Try every test case without copy_grants enabled (default), # and with copy_grants enabled (this base class) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index b97be0ac2..316976955 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -5,6 +5,9 @@ from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +pytestmark = pytest.mark.high_memory + + class TestSnowflakeUnitTestingTypes(BaseUnitTestingTypes): @pytest.fixture def data_types(self): diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py deleted file mode 100644 index eb1a992bd..000000000 --- a/tests/functional/conftest.py +++ /dev/null @@ -1,63 +0,0 @@ -import pathlib - -import pytest - - -TEST_GROUPS = { - "catalog_tests": "last_year", - "column_types": "last_year", - "custom_schema_tests": "last_year", - "dbt_clone": "last_quarter", - "dbt_show": "last_quarter", - "empty": "last_quarter", - "incremental": "microbatch", - "list_relations_tests": "last_month", - "python_model_tests": "python_model", - "query_comment_tests": "last_year", - "simple_copy": "last_year", - "simple_seed": "last_year", - "statement_test": "last_year", - "store_test_failures_tests": "last_year", - "unit_testing": "last_year", - "utils": "last_year", - "test_aliases.py": "last_year", - "test_anonymous_usage_stats.py": "last_year", - "test_basic.py": "last_year", - "test_caching.py": "last_year", - "test_changing_relation_type.py": "large", - "test_concurrency.py": "last_year", - "test_constraints.py": "last_year", - "test_ephemeral.py": "last_year", - "test_get_last_relation_modified.py": "last_year", - "test_grants.py": "last_year", - "test_incremental_microbatch.py": "microbatch", - "test_persist_docs.py": "last_year", - "test_python_model.py": "python_model", - "test_simple_snapshot.py": "last_year", - "test_timestamps.py": "last_year", - "auth_tests": "last_month", - "generic_test_tests": "last_month", - "iceberg": "last_month", - "override_database": "last_year", - "query_tag": "last_month", - "redact_log_values": "last_year", - "relation_tests": "last_month", - "snowflake_view_dependency": "last_year", - "warehouse_test": "last_quarter", - "test_isolated_begin_commit.py": "last_year", -} - - -def pytest_collection_modifyitems(config, items): - test_root = pathlib.Path(config.rootdir) / "tests" / "functional" - for item in items: - try: - test_path = pathlib.Path(item.fspath).relative_to(test_root) - except ValueError: - continue - test_module = test_path.parts[0] - if test_module == "adapter": - test_module = test_path.parts[1] - if mark_name := TEST_GROUPS.get(test_module): - mark = getattr(pytest.mark, mark_name) - item.add_marker(mark) From eb06873b902d05137c8066300a7483a9d5f2a478 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 17 Dec 2024 20:10:39 -0500 Subject: [PATCH 59/62] mark high memory tests and run as a separate pytest run --- tests/functional/relation_tests/test_relation_type_change.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/relation_tests/test_relation_type_change.py b/tests/functional/relation_tests/test_relation_type_change.py index 1024a92ca..5f093b568 100644 --- a/tests/functional/relation_tests/test_relation_type_change.py +++ b/tests/functional/relation_tests/test_relation_type_change.py @@ -9,6 +9,9 @@ from tests.functional.utils import describe_dynamic_table, query_relation_type, update_model +pytestmark = pytest.mark.high_memory + + @dataclass class Model: model: str From 40ec4b5d174ef527f681ed56a560a08f965fa8ff Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 19 Dec 2024 17:22:34 -0500 Subject: [PATCH 60/62] fix build targets --- hatch.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hatch.toml b/hatch.toml index 35410dc3f..23ebd7070 100644 --- a/hatch.toml +++ b/hatch.toml @@ -2,13 +2,14 @@ path = "dbt/adapters/snowflake/__version__.py" [build.targets.sdist] -packages = ["dbt"] +packages = ["src/dbt"] +sources = ["src"] [build.targets.wheel] -packages = ["dbt"] +packages = ["src/dbt"] +sources = ["src"] [envs.default] -python = "3.9" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", @@ -22,6 +23,7 @@ dependencies = [ "pytest-dotenv~=0.5.2", "pytest-logbook~=1.2", "pytest-xdist~=3.6", + "tox~=4.16", # does this pin deps transitively? ] [envs.default.scripts] From c6f90e7ed221bd8ad43c5059123c6f04bf214df3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 19 Dec 2024 17:46:19 -0500 Subject: [PATCH 61/62] remove high_memory marker --- hatch.toml | 5 +---- pyproject.toml | 1 - tests/functional/adapter/dbt_clone/test_dbt_clone.py | 3 --- tests/functional/adapter/test_constraints.py | 3 --- tests/functional/adapter/test_grants.py | 3 --- tests/functional/adapter/unit_testing/test_unit_testing.py | 3 --- tests/functional/relation_tests/test_relation_type_change.py | 3 --- 7 files changed, 1 insertion(+), 20 deletions(-) diff --git a/hatch.toml b/hatch.toml index 23ebd7070..2377e5d6c 100644 --- a/hatch.toml +++ b/hatch.toml @@ -30,10 +30,7 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" -integration-tests = [ - '- python -m pytest {args:tests/functional} -m "not high_memory"', - '- python -m pytest {args:tests/functional} -m "high_memory"', -] +integration-tests = "- python -m pytest {args:tests/functional}" docker-dev = [ "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", "docker run --rm -it --name dbt-snowflake-dev -v $(pwd):/opt/code dbt-snowflake-dev", diff --git a/pyproject.toml b/pyproject.toml index faab5c084..568aa3533 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,4 +46,3 @@ addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] -markers = ["high_memory"] diff --git a/tests/functional/adapter/dbt_clone/test_dbt_clone.py b/tests/functional/adapter/dbt_clone/test_dbt_clone.py index b7911ac76..2a73eb7f4 100644 --- a/tests/functional/adapter/dbt_clone/test_dbt_clone.py +++ b/tests/functional/adapter/dbt_clone/test_dbt_clone.py @@ -6,9 +6,6 @@ from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseClonePossible -pytestmark = pytest.mark.high_memory - - class TestSnowflakeClonePossible(BaseClonePossible): @pytest.fixture(autouse=True) def clean_up(self, project): diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 2ba7c2667..f645bc439 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -19,9 +19,6 @@ ) -pytestmark = pytest.mark.high_memory - - my_model_contract_sql_header_sql = """ {{ config( diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 9cfde9990..30e687f59 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -6,9 +6,6 @@ from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants -pytestmark = pytest.mark.high_memory - - class BaseCopyGrantsSnowflake: # Try every test case without copy_grants enabled (default), # and with copy_grants enabled (this base class) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 316976955..b97be0ac2 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -5,9 +5,6 @@ from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput -pytestmark = pytest.mark.high_memory - - class TestSnowflakeUnitTestingTypes(BaseUnitTestingTypes): @pytest.fixture def data_types(self): diff --git a/tests/functional/relation_tests/test_relation_type_change.py b/tests/functional/relation_tests/test_relation_type_change.py index 5f093b568..1024a92ca 100644 --- a/tests/functional/relation_tests/test_relation_type_change.py +++ b/tests/functional/relation_tests/test_relation_type_change.py @@ -9,9 +9,6 @@ from tests.functional.utils import describe_dynamic_table, query_relation_type, update_model -pytestmark = pytest.mark.high_memory - - @dataclass class Model: model: str From 594de0893c20022a3e152f1248e49e87019c73ab Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 19 Dec 2024 17:48:27 -0500 Subject: [PATCH 62/62] reduce pr noise --- tests/functional/adapter/test_constraints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index f645bc439..03adc3bed 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -18,7 +18,6 @@ model_contract_header_schema_yml, ) - my_model_contract_sql_header_sql = """ {{ config(