From 7fb4549abc52d53dfdbf02da423ee58cc8696ccc Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 31 Jul 2024 21:48:15 -0400 Subject: [PATCH 01/41] Integration workflow update to support all-in-one adapter testing (#1149) --- .github/scripts/update_dbt_core_branch.sh | 20 ------- .../scripts/update_dev_dependency_branches.sh | 21 ++++++++ .github/workflows/integration.yml | 53 +++++++++++++++---- 3 files changed, 64 insertions(+), 30 deletions(-) delete mode 100755 .github/scripts/update_dbt_core_branch.sh create mode 100755 .github/scripts/update_dev_dependency_branches.sh diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh new file mode 100755 index 000000000..022df6a8a --- /dev/null +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e +set -e + + +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file +else + sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c566a20f0..b3662d5c0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,8 @@ name: Adapter Integration Tests +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" + on: # pushes to release branches push: @@ -34,10 +36,31 @@ on: # manual trigger workflow_dispatch: inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" + name: + description: "Name to associate with run (example: 'dbt-adapters-242')" required: false type: string + default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: "main" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" + type: string + required: false + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -141,12 +164,19 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' + if: github.event_name == 'push' + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false + ref: ${{ inputs.adapter_branch }} - # explicity checkout the branch for the PR, + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' @@ -160,6 +190,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + ./.github/scripts/update_dev_dependency_branches.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} + cat dev-requirements.txt + - name: Install python dependencies run: | python -m pip install --user --upgrade pip @@ -167,12 +206,6 @@ jobs: python -m pip --version tox --version - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - name: Run tox (snowflake) if: matrix.adapter == 'snowflake' env: From 3fbc0749491f40d34014336457753b140e1fb1ee Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:08:00 -0400 Subject: [PATCH 02/41] Isolating distribution testing (#1161) * Splitting up distribution testing * Adding back in if statement * Adding changie file * Fixing whitespace --- .../Under the Hood-20240806-215935.yaml | 6 ++++++ .github/workflows/main.yml | 17 +++++------------ 2 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240806-215935.yaml diff --git a/.changes/unreleased/Under the Hood-20240806-215935.yaml b/.changes/unreleased/Under the Hood-20240806-215935.yaml new file mode 100644 index 000000000..660918350 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240806-215935.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Isolating distribution testing +time: 2024-08-06T21:59:35.284641-04:00 +custom: + Author: leahwicz + Issue: "1130" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 67b8d4e57..95ce18033 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -163,7 +163,7 @@ jobs: overwrite: true test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} + name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} if: needs.build.outputs.is_alpha == 0 @@ -176,6 +176,7 @@ jobs: matrix: os: [ubuntu-latest, macos-12, windows-latest] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + dist-type: ['whl', 'gz'] steps: - name: Set up Python ${{ matrix.python-version }} @@ -197,18 +198,10 @@ jobs: - name: Show distributions run: ls -lh dist/ - - name: Install wheel distributions + - name: Install ${{ matrix.dist-type }} distributions run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check wheel distributions - run: | - python -c "import dbt.adapters.snowflake" - - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check source distributions + - name: Check ${{ matrix.dist-type }} distributions run: | python -c "import dbt.adapters.snowflake" From 49623d7309bb64600d2ddb3f545ed6a9d8a0ddaf Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:26:43 -0700 Subject: [PATCH 03/41] Add support for Iceberg Table Materialization (#1170) * Add materializations of table and dynamic table. * Add the method to tell something is iceberg format and pipe that through to relation object * Finish create macro and fix alters. * Finish todo items and begin cleaning code. * revert dynamic table changes. * Fix the drop by fixing snowflake__show_iceberg_relations * Transient needs sophisticated handling based on what user specifies for transient manually. * Try to figure out what the right None semantics are. * Revert to original statement. * Fix the transient behavior by passing table_type again. * Rename object_format config param to table_format * Migrate Jinja macros to Python. * All classes are frozen * Clean up the metadata queries that power is_iceberg column generation * Fix Python models generation argument * Add changelog. * Try to fix duplication of join record issues. * Use the RelationConfig protocol for type checking. * Fix transient semantics. * Add functional tests. * Fix test. * Fix test. * Fix test and remove strip calls * Add view test case. * Code review comments. * I'm using too new a version of mypy for Self. * Add a behavior flag for iceberg table materialization. * Flip order of flag. * Adjust test. --------- Co-authored-by: Mila Page --- .../unreleased/Features-20240911-001806.yaml | 6 + dbt/adapters/snowflake/impl.py | 41 +++++- dbt/adapters/snowflake/relation.py | 132 +++++++++++++++++- .../snowflake/relation_configs/__init__.py | 1 + .../snowflake/relation_configs/formats.py | 14 ++ dbt/include/snowflake/macros/adapters.sql | 27 ++-- .../macros/materializations/table.sql | 15 +- .../macros/relations/table/create.sql | 33 +++-- tests/functional/iceberg/test_table_basic.py | 106 ++++++++++++++ 9 files changed, 339 insertions(+), 36 deletions(-) create mode 100644 .changes/unreleased/Features-20240911-001806.yaml create mode 100644 dbt/adapters/snowflake/relation_configs/formats.py create mode 100644 tests/functional/iceberg/test_table_basic.py diff --git a/.changes/unreleased/Features-20240911-001806.yaml b/.changes/unreleased/Features-20240911-001806.yaml new file mode 100644 index 000000000..024480b96 --- /dev/null +++ b/.changes/unreleased/Features-20240911-001806.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for Iceberg table materializations. +time: 2024-09-11T00:18:06.780586-07:00 +custom: + Author: versusfacit + Issue: "321" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 6854b199d..7e8ec9cf2 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -9,6 +9,7 @@ LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, ) +from dbt_common.behavior_flags import BehaviorFlag from dbt_common.contracts.constraints import ConstraintType from dbt_common.contracts.metadata import ( TableMetadata, @@ -20,7 +21,10 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values -from dbt.adapters.snowflake.relation_configs import SnowflakeRelationType +from dbt.adapters.snowflake.relation_configs import ( + SnowflakeRelationType, + TableFormat, +) from dbt.adapters.snowflake import SnowflakeColumn from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -44,6 +48,11 @@ class SnowflakeConfig(AdapterConfig): merge_update_columns: Optional[str] = None target_lag: Optional[str] = None + # extended formats + table_format: Optional[str] = None + external_volume: Optional[str] = None + base_location_subpath: Optional[str] = None + class SnowflakeAdapter(SQLAdapter): Relation = SnowflakeRelation @@ -69,6 +78,10 @@ class SnowflakeAdapter(SQLAdapter): } ) + @property + def _behavior_flags(self) -> List[BehaviorFlag]: + return [{"name": "enable_iceberg_materializations", "default": False}] + @classmethod def date_function(cls): return "CURRENT_TIMESTAMP()" @@ -223,8 +236,9 @@ def list_relations_without_caching( self, schema_relation: SnowflakeRelation ) -> List[SnowflakeRelation]: kwargs = {"schema_relation": schema_relation} + try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + schema_objects = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) except DbtDatabaseError as exc: # if the schema doesn't exist, we just want to return. # Alternatively, we could query the list of schemas before we start @@ -235,18 +249,26 @@ def list_relations_without_caching( # this can be reduced to always including `is_dynamic` once bundle `2024_03` is mandatory columns = ["database_name", "schema_name", "name", "kind"] - if "is_dynamic" in results.column_names: + if "is_dynamic" in schema_objects.column_names: columns.append("is_dynamic") + if "is_iceberg" in schema_objects.column_names: + columns.append("is_iceberg") - return [self._parse_list_relations_result(result) for result in results.select(columns)] + return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)] def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation: # this can be reduced to always including `is_dynamic` once bundle `2024_03` is mandatory + # this can be reduced to always including `is_iceberg` once Snowflake adds it to show objects try: - database, schema, identifier, relation_type, is_dynamic = result + if self.behavior.enable_iceberg_materializations.no_warn: + database, schema, identifier, relation_type, is_dynamic, is_iceberg = result + else: + database, schema, identifier, relation_type, is_dynamic = result except ValueError: database, schema, identifier, relation_type = result is_dynamic = "N" + if self.behavior.enable_iceberg_materializations.no_warn: + is_iceberg = "N" try: relation_type = self.Relation.get_relation_type(relation_type.lower()) @@ -256,12 +278,21 @@ def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation if relation_type == self.Relation.Table and is_dynamic == "Y": relation_type = self.Relation.DynamicTable + # This line is the main gate on supporting Iceberg materializations. Pass forward a default + # table format, and no downstream table macros can build iceberg relations. + table_format: str = ( + TableFormat.ICEBERG + if self.behavior.enable_iceberg_materializations.no_warn and is_iceberg in ("Y", "YES") + else TableFormat.DEFAULT + ) quote_policy = {"database": True, "schema": True, "identifier": True} + return self.Relation.create( database=database, schema=schema, identifier=identifier, type=relation_type, + table_format=table_format, quote_policy=quote_policy, ) diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index ace85695b..224b2b75e 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -1,8 +1,12 @@ +import textwrap + from dataclasses import dataclass, field -from typing import FrozenSet, Optional, Type +from typing import FrozenSet, Optional, Type, Iterator, Tuple + from dbt.adapters.base.relation import BaseRelation from dbt.adapters.contracts.relation import ComponentName, RelationConfig +from dbt.adapters.events.types import AdapterEventWarning, AdapterEventDebug from dbt.adapters.relation_configs import ( RelationConfigBase, RelationConfigChangeAction, @@ -10,6 +14,7 @@ ) from dbt.adapters.utils import classproperty from dbt_common.exceptions import DbtRuntimeError +from dbt_common.events.functions import fire_event, warn_or_error from dbt.adapters.snowflake.relation_configs import ( SnowflakeDynamicTableConfig, @@ -17,6 +22,7 @@ SnowflakeDynamicTableRefreshModeConfigChange, SnowflakeDynamicTableTargetLagConfigChange, SnowflakeDynamicTableWarehouseConfigChange, + TableFormat, SnowflakeQuotePolicy, SnowflakeRelationType, ) @@ -25,6 +31,7 @@ @dataclass(frozen=True, eq=False, repr=False) class SnowflakeRelation(BaseRelation): type: Optional[SnowflakeRelationType] = None + table_format: str = TableFormat.DEFAULT quote_policy: SnowflakeQuotePolicy = field(default_factory=lambda: SnowflakeQuotePolicy()) require_alias: bool = False relation_configs = { @@ -53,6 +60,10 @@ class SnowflakeRelation(BaseRelation): def is_dynamic_table(self) -> bool: return self.type == SnowflakeRelationType.DynamicTable + @property + def is_iceberg_format(self) -> bool: + return self.table_format == TableFormat.ICEBERG + @classproperty def DynamicTable(cls) -> str: return str(SnowflakeRelationType.DynamicTable) @@ -120,3 +131,122 @@ def as_case_sensitive(self) -> "SnowflakeRelation": path_part_map[path] = part.upper() return self.replace_path(**path_part_map) + + def get_ddl_prefix_for_create(self, config: RelationConfig, temporary: bool) -> str: + """ + This macro renders the appropriate DDL prefix during the create_table_as + macro. It decides based on mutually exclusive table configuration options: + + - TEMPORARY: Indicates a table that exists only for the duration of the session. + - ICEBERG: A specific storage format that requires a distinct DDL layout. + - TRANSIENT: A table similar to a permanent table but without fail-safe. + + Additional Caveats for Iceberg models: + - transient=true throws a warning because Iceberg does not support transient tables + - A temporary relation is never an Iceberg relation because Iceberg does not + support temporary relations. + """ + + transient_explicitly_set_true: bool = config.get("transient", False) + + # Temporary tables are a Snowflake feature that do not exist in the + # Iceberg framework. We ignore the Iceberg status of the model. + if temporary: + return "temporary" + elif self.is_iceberg_format: + # Log a warning that transient=true on an Iceberg relation is ignored. + if transient_explicitly_set_true: + warn_or_error( + AdapterEventWarning( + base_msg=( + "Iceberg format relations cannot be transient. Please " + "remove either the transient or iceberg config options " + f"from {self.path.database}.{self.path.schema}." + f"{self.path.identifier}. If left unmodified, dbt will " + "ignore 'transient'." + ) + ) + ) + + return "iceberg" + + # Always supply transient on table create DDL unless user specifically sets + # transient to false or unset. Might as well update the object attribute too! + elif transient_explicitly_set_true or config.get("transient", True): + return "transient" + else: + return "" + + def get_ddl_prefix_for_alter(self) -> str: + """All ALTER statements on Iceberg tables require an ICEBERG prefix""" + if self.is_iceberg_format: + return "iceberg" + else: + return "" + + def get_iceberg_ddl_options(self, config: RelationConfig) -> str: + base_location: str = f"_dbt/{self.schema}/{self.name}" + + if subpath := config.get("base_location_subpath"): + base_location += f"/{subpath}" + + iceberg_ddl_predicates: str = f""" + external_volume = '{config.get('external_volume')}' + catalog = 'snowflake' + base_location = '{base_location}' + """ + return textwrap.indent(textwrap.dedent(iceberg_ddl_predicates), " " * 10) + + def __drop_conditions(self, old_relation: "SnowflakeRelation") -> Iterator[Tuple[bool, str]]: + drop_view_message: str = ( + f"Dropping relation {old_relation} because it is a view and target relation {self} " + f"is of type {self.type}." + ) + + drop_table_for_iceberg_message: str = ( + f"Dropping relation {old_relation} because it is a default format table " + f"and target relation {self} is an Iceberg format table." + ) + + drop_iceberg_for_table_message: str = ( + f"Dropping relation {old_relation} because it is an Iceberg format table " + f"and target relation {self} is a default format table." + ) + + # An existing view must be dropped for model to build into a table". + yield (not old_relation.is_table, drop_view_message) + # An existing table must be dropped for model to build into an Iceberg table. + yield ( + old_relation.is_table + and not old_relation.is_iceberg_format + and self.is_iceberg_format, + drop_table_for_iceberg_message, + ) + # existing Iceberg table must be dropped for model to build into a table. + yield ( + old_relation.is_table + and old_relation.is_iceberg_format + and not self.is_iceberg_format, + drop_iceberg_for_table_message, + ) + + def needs_to_drop(self, old_relation: Optional["SnowflakeRelation"]) -> bool: + """ + To convert between Iceberg and non-Iceberg relations, a preemptive drop is + required. + + drops cause latency, but it should be a relatively infrequent occurrence. + + Some Boolean expression below are logically redundant, but this is done for easier + readability. + """ + + if old_relation is None: + return False + + for condition, message in self.__drop_conditions(old_relation): + if condition: + fire_event(AdapterEventDebug(base_msg=message)) + return True + + return False diff --git a/dbt/adapters/snowflake/relation_configs/__init__.py b/dbt/adapters/snowflake/relation_configs/__init__.py index 62f95faff..61941ab50 100644 --- a/dbt/adapters/snowflake/relation_configs/__init__.py +++ b/dbt/adapters/snowflake/relation_configs/__init__.py @@ -10,3 +10,4 @@ SnowflakeQuotePolicy, SnowflakeRelationType, ) +from dbt.adapters.snowflake.relation_configs.formats import TableFormat diff --git a/dbt/adapters/snowflake/relation_configs/formats.py b/dbt/adapters/snowflake/relation_configs/formats.py new file mode 100644 index 000000000..460241d9d --- /dev/null +++ b/dbt/adapters/snowflake/relation_configs/formats.py @@ -0,0 +1,14 @@ +from dbt_common.dataclass_schema import StrEnum # doesn't exist in standard library until py3.11 + + +class TableFormat(StrEnum): + """ + Snowflake docs refers to this an 'Object Format.' + Data practitioners and interfaces refer to this as 'Table Format's, hence the term's use here. + """ + + DEFAULT = "default" + ICEBERG = "iceberg" + + def __str__(self): + return self.value diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 4cb4bcffa..aa8895819 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -137,15 +137,24 @@ {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} {%- set max_total_results = max_results_per_iter * max_iter -%} - {% if schema_relation is string %} - {%- set sql -%} - show objects in {{ schema_relation }} limit {{ max_results_per_iter }} - {%- endset -%} - {% else %} - {%- set sql -%} - show objects in {{ schema_relation.include(identifier=False) }} limit {{ max_results_per_iter }} - {%- endset -%} - {% endif -%} + {%- set sql -%} + {% if schema_relation is string %} + show objects in {{ schema_relation }} limit {{ max_results_per_iter }}; + {% else %} + show objects in {{ schema_relation.include(identifier=False) }} limit {{ max_results_per_iter }}; + {% endif -%} + + {# -- Gated for performance reason. If you don't want Iceberg, you shouldn't pay the + -- latency penalty. #} + {% if adapter.behavior.enable_iceberg_materializations.no_warn %} + select all_objects.*, is_iceberg as "is_iceberg" + from table(result_scan(last_query_id(-1))) all_objects + left join INFORMATION_SCHEMA.tables as all_tables + on all_tables.table_name = all_objects."name" + and all_tables.table_schema = all_objects."schema_name" + and all_tables.table_catalog = all_objects."database_name" + {% endif -%} + {%- endset -%} {%- set result = run_query(sql) -%} diff --git a/dbt/include/snowflake/macros/materializations/table.sql b/dbt/include/snowflake/macros/materializations/table.sql index ef201c705..cbc6d9ce6 100644 --- a/dbt/include/snowflake/macros/materializations/table.sql +++ b/dbt/include/snowflake/macros/materializations/table.sql @@ -8,16 +8,17 @@ {% set grant_config = config.get('grants') %} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, - schema=schema, - database=database, type='table') -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, + schema=schema, + database=database, + type='table', + table_format=config.get('table_format', 'default') + ) -%} {{ run_hooks(pre_hooks) }} - {#-- Drop the relation if it was a view to "convert" it in a table. This may lead to - -- downtime, but it should be a relatively infrequent occurrence #} - {% if old_relation is not none and not old_relation.is_table %} - {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} + {% if target_relation.needs_to_drop(old_relation) %} {{ drop_relation_if_exists(old_relation) }} {% endif %} diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index c6bc8f775..355150e28 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,14 +1,8 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} - {%- set transient = config.get('transient', default=true) -%} - - {% if temporary -%} - {%- set table_type = "temporary" -%} - {%- elif transient -%} - {%- set table_type = "transient" -%} - {%- else -%} - {%- set table_type = "" -%} - {%- endif %} + {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} + {%- set alter_prefix = relation.get_ddl_prefix_for_alter() -%} + {# Generate DDL/DML #} {%- if language == 'sql' -%} {%- set cluster_by_keys = config.get('cluster_by', default=none) -%} {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%} @@ -26,7 +20,15 @@ {{ sql_header if sql_header is not none }} - create or replace {{ table_type }} table {{ relation }} + create or replace {{ materialization_prefix }} table {{ relation }} + {%- if relation.is_iceberg_format %} + {# + Valid DDL in CTAS statements. Plain create statements have a different order. + https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table + #} + {{ relation.get_iceberg_ddl_options(config.model.config) }} + {%- endif -%} + {%- set contract_config = config.get('contract') -%} {%- if contract_config.enforced -%} {{ get_assert_columns_equivalent(sql) }} @@ -44,14 +46,17 @@ {%- endif %} ); {% if cluster_by_string is not none and not temporary -%} - alter table {{relation}} cluster by ({{cluster_by_string}}); + alter {{ alter_prefix }} table {{relation}} cluster by ({{cluster_by_string}}); {%- endif -%} - {% if enable_automatic_clustering and cluster_by_string is not none and not temporary -%} - alter table {{relation}} resume recluster; + {% if enable_automatic_clustering and cluster_by_string is not none and not temporary %} + alter {{ alter_prefix }} table {{relation}} resume recluster; {%- endif -%} {%- elif language == 'python' -%} - {{ py_write_table(compiled_code=compiled_code, target_relation=relation, table_type=table_type) }} + {%- if relation.is_iceberg_format %} + {% do exceptions.raise_compiler_error('Iceberg is incompatible with Python models. Please use a SQL model for the iceberg format.') %} + {%- endif %} + {{ py_write_table(compiled_code=compiled_code, target_relation=relation, table_type=relation.get_ddl_prefix_for_create(config.model.config, temporary)) }} {%- else -%} {% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %} {%- endif -%} diff --git a/tests/functional/iceberg/test_table_basic.py b/tests/functional/iceberg/test_table_basic.py new file mode 100644 index 000000000..0bfdf59f1 --- /dev/null +++ b/tests/functional/iceberg/test_table_basic.py @@ -0,0 +1,106 @@ +import pytest + +from pathlib import Path + +from dbt.tests.util import run_dbt, rm_file, write_file + +_MODEL_BASIC_TABLE_MODEL = """ +{{ + config( + materialized = "table", + cluster_by=['id'], + ) +}} +select 1 as id +""" + +_MODEL_BASIC_ICEBERG_MODEL = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BUILT_ON_ICEBERG_TABLE = """ +{{ + config( + materialized = "table", + ) +}} +select * from {{ ref('iceberg_table') }} +""" + +_MODEL_TABLE_BEFORE_SWAP = """ +{{ + config( + materialized = "table", + ) +}} +select 1 as id +""" + +_MODEL_VIEW_BEFORE_SWAP = """ +select 1 as id +""" + +_MODEL_TABLE_FOR_SWAP_ICEBERG = """ +{{ + config( + materialized = "table", + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", + ) +}} +select 1 as id +""" + + +class TestIcebergTableBuilds: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def models(self): + return { + "first_table.sql": _MODEL_BASIC_TABLE_MODEL, + "iceberg_table.sql": _MODEL_BASIC_ICEBERG_MODEL, + "table_built_on_iceberg_table.sql": _MODEL_BUILT_ON_ICEBERG_TABLE, + } + + def test_iceberg_tables_build_and_can_be_referred(self, project): + run_results = run_dbt() + assert len(run_results) == 3 + + +class TestIcebergTableTypeBuildsOnExistingTable: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.mark.parametrize("start_model", [_MODEL_TABLE_BEFORE_SWAP, _MODEL_VIEW_BEFORE_SWAP]) + def test_changing_model_types(self, project, start_model): + model_file = project.project_root / Path("models") / Path("my_model.sql") + + write_file(start_model, model_file) + run_results = run_dbt() + assert len(run_results) == 1 + + rm_file(model_file) + write_file(_MODEL_TABLE_FOR_SWAP_ICEBERG, model_file) + run_results = run_dbt() + assert len(run_results) == 1 + + rm_file(model_file) + write_file(start_model, model_file) + run_results = run_dbt() + assert len(run_results) == 1 From 3cbe12f4543d0357ec0ed4b7f6a17345c6198bae Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 18 Sep 2024 13:29:42 -0400 Subject: [PATCH 04/41] Microbatch strategy (#1179) * first pass: add incremental_predicates * safely add incremental_predicates + testing * remove requirement for unique_id --------- Co-authored-by: Quigley Malcolm --- .../unreleased/Features-20240913-215416.yaml | 6 ++++ dbt/adapters/snowflake/impl.py | 2 +- .../macros/materializations/merge.sql | 32 +++++++++++++++++++ .../adapter/test_incremental_microbatch.py | 24 ++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20240913-215416.yaml create mode 100644 tests/functional/adapter/test_incremental_microbatch.py diff --git a/.changes/unreleased/Features-20240913-215416.yaml b/.changes/unreleased/Features-20240913-215416.yaml new file mode 100644 index 000000000..b2a6e556e --- /dev/null +++ b/.changes/unreleased/Features-20240913-215416.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Microbatch incremental strategy +time: 2024-09-13T21:54:16.492885-04:00 +custom: + Author: michelleark + Issue: "1182" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 7e8ec9cf2..a6297887d 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -416,7 +416,7 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): return response def valid_incremental_strategies(self): - return ["append", "merge", "delete+insert"] + return ["append", "merge", "delete+insert", "microbatch"] def debug_query(self): """Override for DebugTask method""" diff --git a/dbt/include/snowflake/macros/materializations/merge.sql b/dbt/include/snowflake/macros/materializations/merge.sql index e93b29155..57c58afdd 100644 --- a/dbt/include/snowflake/macros/materializations/merge.sql +++ b/dbt/include/snowflake/macros/materializations/merge.sql @@ -48,3 +48,35 @@ {% set dml = default__get_incremental_append_sql(get_incremental_append_sql) %} {% do return(snowflake_dml_explicit_transaction(dml)) %} {% endmacro %} + + +{% macro snowflake__get_incremental_microbatch_sql(arg_dict) %} + {%- set target = arg_dict["target_relation"] -%} + {%- set source = arg_dict["temp_relation"] -%} + {%- set dest_columns = arg_dict["dest_columns"] -%} + {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%} + + {#-- Add additional incremental_predicates to filter for batch --#} + {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= TIMESTAMP '" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "'") %} + {% endif %} + {% if model.config.__dbt_internal_microbatch_event_time_end -%} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < TIMESTAMP '" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "'") %} + {% endif %} + {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} + + delete from {{ target }} DBT_INTERNAL_TARGET + using {{ source }} + where ( + {% for predicate in incremental_predicates %} + {%- if not loop.first %}and {% endif -%} {{ predicate }} + {% endfor %} + ); + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ) +{% endmacro %} diff --git a/tests/functional/adapter/test_incremental_microbatch.py b/tests/functional/adapter/test_incremental_microbatch.py new file mode 100644 index 000000000..bbb57f96c --- /dev/null +++ b/tests/functional/adapter/test_incremental_microbatch.py @@ -0,0 +1,24 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_microbatch import ( + BaseMicrobatch, +) + + +# No requirement for a unique_id for snowflake microbatch! +_microbatch_model_no_unique_id_sql = """ +{{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day') }} +select * from {{ ref('input_model') }} +""" + + +class TestSnowflakeMicrobatch(BaseMicrobatch): + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return _microbatch_model_no_unique_id_sql + + @pytest.fixture(scope="class") + def insert_two_rows_sql(self, project) -> str: + test_schema_relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + return f"insert into {test_schema_relation}.input_model (id, event_time) values (4, '2020-01-04 00:00:00-0'), (5, '2020-01-05 00:00:00-0')" From 0bfef5b4abf27e1179867d039c45b33bd12be626 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:11:07 -0700 Subject: [PATCH 05/41] Throw error when building Iceberg tables without behavior flag set (#1184) * pipe the behavior through to relation in a temporary way to help with warnings in cli * Adjust flag to new dbt-common setup. * add changelog. * Throw a compiler warning for iceberg model build without flag. --------- Co-authored-by: VersusFacit Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Under the Hood-20240917-181147.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 13 ++++++++++++- .../snowflake/macros/relations/table/create.sql | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20240917-181147.yaml diff --git a/.changes/unreleased/Under the Hood-20240917-181147.yaml b/.changes/unreleased/Under the Hood-20240917-181147.yaml new file mode 100644 index 000000000..2f52174dd --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240917-181147.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Change behavior flag semantics to log iceberg flag warnings.. +time: 2024-09-17T18:11:47.525026-07:00 +custom: + Author: versusfacit + Issue: "321" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index a6297887d..69da11802 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -80,7 +80,18 @@ class SnowflakeAdapter(SQLAdapter): @property def _behavior_flags(self) -> List[BehaviorFlag]: - return [{"name": "enable_iceberg_materializations", "default": False}] + return [ + { + "name": "enable_iceberg_materializations", + "default": False, + "description": ( + "Enabling Iceberg materializations introduces latency to metadata queries, " + "specifically within the list_relations_without_caching macro. Since Iceberg " + "benefits only those actively using it, we've made this behavior opt-in to " + "prevent unnecessary latency for other users." + ), + } + ] @classmethod def date_function(cls): diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index 355150e28..e60b93039 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,4 +1,9 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} + + {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to .') %} + {%- endif %} + {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} {%- set alter_prefix = relation.get_ddl_prefix_for_alter() -%} From 084674f68f52216a48e5c70a202666d3c2de1af4 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 19 Sep 2024 10:54:19 -0400 Subject: [PATCH 06/41] Remove a macro that is entirely a docstring (#1185) * remove a macro that only contains a docstring --- .../snowflake/macros/materializations/table.sql | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/table.sql b/dbt/include/snowflake/macros/materializations/table.sql index cbc6d9ce6..9ee8a0b12 100644 --- a/dbt/include/snowflake/macros/materializations/table.sql +++ b/dbt/include/snowflake/macros/materializations/table.sql @@ -72,17 +72,3 @@ def main(session): materialize(session, df, dbt.this) return "OK" {% endmacro %} - -{% macro py_script_comment()%} -# To run this in snowsight, you need to select entry point to be main -# And you may have to modify the return type to text to get the result back -# def main(session): -# dbt = dbtObj(session.table) -# df = model(dbt, session) -# return df.collect() - -# to run this in local notebook, you need to create a session following examples https://github.com/Snowflake-Labs/sfguide-getting-started-snowpark-python -# then you can do the following to run model -# dbt = dbtObj(session.table) -# df = model(dbt, session) -{%endmacro%} From 34c44422845f8cc720821b8fa481733891a450bc Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:18:59 -0400 Subject: [PATCH 07/41] Dynamic Table testing improvements (#1187) * move dynamic table tests down into the relation tests folder * make utils more generic, move out of dynamic table tests * add init files for namespacing in test discovery * remove 2024_03 bundle items --- dbt/adapters/snowflake/impl.py | 35 +- .../test_dynamic_tables_basic.py | 186 ----------- .../test_dynamic_tables_changes.py | 307 ------------------ .../adapter/dynamic_table_tests/utils.py | 53 --- tests/functional/relation_tests/__init__.py | 0 .../dynamic_table_tests/__init__.py | 0 .../dynamic_table_tests/models.py | 50 +++ .../dynamic_table_tests/test_basic.py | 30 ++ .../test_configuration_changes.py | 103 ++++++ .../files.py => relation_tests/models.py} | 10 +- .../test_relation_type_change.py | 64 ++++ tests/functional/utils.py | 78 +++++ 12 files changed, 341 insertions(+), 575 deletions(-) delete mode 100644 tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_basic.py delete mode 100644 tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py delete mode 100644 tests/functional/adapter/dynamic_table_tests/utils.py create mode 100644 tests/functional/relation_tests/__init__.py create mode 100644 tests/functional/relation_tests/dynamic_table_tests/__init__.py create mode 100644 tests/functional/relation_tests/dynamic_table_tests/models.py create mode 100644 tests/functional/relation_tests/dynamic_table_tests/test_basic.py create mode 100644 tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py rename tests/functional/{adapter/dynamic_table_tests/files.py => relation_tests/models.py} (78%) create mode 100644 tests/functional/relation_tests/test_relation_type_change.py create mode 100644 tests/functional/utils.py diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 69da11802..5b5881eed 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -258,28 +258,20 @@ def list_relations_without_caching( return [] raise - # this can be reduced to always including `is_dynamic` once bundle `2024_03` is mandatory - columns = ["database_name", "schema_name", "name", "kind"] - if "is_dynamic" in schema_objects.column_names: - columns.append("is_dynamic") - if "is_iceberg" in schema_objects.column_names: + # this can be collapsed once Snowflake adds is_iceberg to show objects + columns = ["database_name", "schema_name", "name", "kind", "is_dynamic"] + if self.behavior.enable_iceberg_materializations.no_warn: columns.append("is_iceberg") return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)] def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation: - # this can be reduced to always including `is_dynamic` once bundle `2024_03` is mandatory - # this can be reduced to always including `is_iceberg` once Snowflake adds it to show objects - try: - if self.behavior.enable_iceberg_materializations.no_warn: - database, schema, identifier, relation_type, is_dynamic, is_iceberg = result - else: - database, schema, identifier, relation_type, is_dynamic = result - except ValueError: - database, schema, identifier, relation_type = result - is_dynamic = "N" - if self.behavior.enable_iceberg_materializations.no_warn: - is_iceberg = "N" + # this can be collapsed once Snowflake adds is_iceberg to show objects + if self.behavior.enable_iceberg_materializations.no_warn: + database, schema, identifier, relation_type, is_dynamic, is_iceberg = result + else: + database, schema, identifier, relation_type, is_dynamic = result + is_iceberg = "N" try: relation_type = self.Relation.get_relation_type(relation_type.lower()) @@ -289,13 +281,8 @@ def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation if relation_type == self.Relation.Table and is_dynamic == "Y": relation_type = self.Relation.DynamicTable - # This line is the main gate on supporting Iceberg materializations. Pass forward a default - # table format, and no downstream table macros can build iceberg relations. - table_format: str = ( - TableFormat.ICEBERG - if self.behavior.enable_iceberg_materializations.no_warn and is_iceberg in ("Y", "YES") - else TableFormat.DEFAULT - ) + table_format = TableFormat.ICEBERG if is_iceberg in ("Y", "YES") else TableFormat.DEFAULT + quote_policy = {"database": True, "schema": True, "identifier": True} return self.Relation.create( diff --git a/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_basic.py b/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_basic.py deleted file mode 100644 index a17f5d267..000000000 --- a/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_basic.py +++ /dev/null @@ -1,186 +0,0 @@ -from typing import Optional, Tuple - -import pytest - -from dbt.tests.util import ( - get_model_file, - run_dbt, - run_dbt_and_capture, - set_model_file, -) - -from dbt.adapters.snowflake.relation import SnowflakeRelation, SnowflakeRelationType -from tests.functional.adapter.dynamic_table_tests.files import ( - MY_DYNAMIC_TABLE, - MY_SEED, - MY_TABLE, - MY_VIEW, -) -from tests.functional.adapter.dynamic_table_tests.utils import query_relation_type - - -class TestSnowflakeDynamicTableBasic: - @staticmethod - def insert_record(project, table: SnowflakeRelation, record: Tuple[int, int]): - my_id, value = record - project.run_sql(f"insert into {table} (id, value) values ({my_id}, {value})") - - @staticmethod - def refresh_dynamic_table(project, dynamic_table: SnowflakeRelation): - sql = f"alter dynamic table {dynamic_table} refresh" - project.run_sql(sql) - - @staticmethod - def query_row_count(project, relation: SnowflakeRelation) -> int: - sql = f"select count(*) from {relation}" - return project.run_sql(sql, fetch="one")[0] - - @staticmethod - def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: - return query_relation_type(project, relation) - - @pytest.fixture(scope="class", autouse=True) - def seeds(self): - return {"my_seed.csv": MY_SEED} - - @pytest.fixture(scope="class", autouse=True) - def models(self): - yield { - "my_table.sql": MY_TABLE, - "my_view.sql": MY_VIEW, - "my_dynamic_table.sql": MY_DYNAMIC_TABLE, - } - - @pytest.fixture(scope="class") - def my_dynamic_table(self, project) -> SnowflakeRelation: - return project.adapter.Relation.create( - identifier="my_dynamic_table", - schema=project.test_schema, - database=project.database, - type=SnowflakeRelationType.DynamicTable, - ) - - @pytest.fixture(scope="class") - def my_view(self, project) -> SnowflakeRelation: - return project.adapter.Relation.create( - identifier="my_view", - schema=project.test_schema, - database=project.database, - type=SnowflakeRelationType.View, - ) - - @pytest.fixture(scope="class") - def my_table(self, project) -> SnowflakeRelation: - return project.adapter.Relation.create( - identifier="my_table", - schema=project.test_schema, - database=project.database, - type=SnowflakeRelationType.Table, - ) - - @pytest.fixture(scope="class") - def my_seed(self, project) -> SnowflakeRelation: - return project.adapter.Relation.create( - identifier="my_seed", - schema=project.test_schema, - database=project.database, - type=SnowflakeRelationType.Table, - ) - - @staticmethod - def load_model(project, current_model, new_model): - model_to_load = get_model_file(project, new_model) - set_model_file(project, current_model, model_to_load) - - @pytest.fixture(scope="function", autouse=True) - def setup(self, project, my_dynamic_table, my_view, my_table): - run_dbt(["seed"]) - run_dbt(["run", "--models", my_dynamic_table.identifier, "--full-refresh"]) - - # the tests touch these files, store their contents in memory - my_dynamic_table_config = get_model_file(project, my_dynamic_table) - my_view_config = get_model_file(project, my_view) - my_table_config = get_model_file(project, my_table) - - yield - - # and then reset them after the test runs - set_model_file(project, my_dynamic_table, my_dynamic_table_config) - set_model_file(project, my_view, my_view_config) - set_model_file(project, my_table, my_table_config) - project.run_sql(f"drop schema if exists {project.test_schema} cascade") - - def test_dynamic_table_create(self, project, my_dynamic_table): - # setup creates it; verify it's there - assert self.query_relation_type(project, my_dynamic_table) == "dynamic_table" - - def test_dynamic_table_create_idempotent(self, project, my_dynamic_table): - # setup creates it once; verify it's there and run once - assert self.query_relation_type(project, my_dynamic_table) == "dynamic_table" - run_dbt(["run", "--models", my_dynamic_table.identifier]) - assert self.query_relation_type(project, my_dynamic_table) == "dynamic_table" - - def test_dynamic_table_full_refresh(self, project, my_dynamic_table): - _, logs = run_dbt_and_capture( - ["--debug", "run", "--models", my_dynamic_table.identifier, "--full-refresh"] - ) - assert self.query_relation_type(project, my_dynamic_table) == "dynamic_table" - - def test_dynamic_table_replaces_table(self, project, my_table, my_dynamic_table): - run_dbt(["run", "--models", my_table.identifier]) - assert self.query_relation_type(project, my_table) == "table" - - self.load_model(project, my_table, my_dynamic_table) - - run_dbt(["run", "--models", my_table.identifier]) - assert self.query_relation_type(project, my_table) == "dynamic_table" - - def test_dynamic_table_replaces_view(self, project, my_view, my_dynamic_table): - run_dbt(["run", "--models", my_view.identifier]) - assert self.query_relation_type(project, my_view) == "view" - - self.load_model(project, my_view, my_dynamic_table) - - run_dbt(["run", "--models", my_view.identifier]) - assert self.query_relation_type(project, my_view) == "dynamic_table" - - def test_table_replaces_dynamic_table(self, project, my_dynamic_table, my_table): - run_dbt(["run", "--models", my_dynamic_table.identifier]) - assert self.query_relation_type(project, my_dynamic_table) == "dynamic_table" - - self.load_model(project, my_dynamic_table, my_table) - - run_dbt(["run", "--models", my_dynamic_table.identifier]) - assert self.query_relation_type(project, my_dynamic_table) == "table" - - def test_view_replaces_dynamic_table(self, project, my_dynamic_table, my_view): - run_dbt(["run", "--models", my_dynamic_table.identifier]) - assert self.query_relation_type(project, my_dynamic_table) == "dynamic_table" - - self.load_model(project, my_dynamic_table, my_view) - - run_dbt(["run", "--models", my_dynamic_table.identifier]) - assert self.query_relation_type(project, my_dynamic_table) == "view" - - def test_dynamic_table_only_updates_after_refresh(self, project, my_dynamic_table, my_seed): - # poll database - table_start = self.query_row_count(project, my_seed) - view_start = self.query_row_count(project, my_dynamic_table) - - # insert new record in table - self.insert_record(project, my_seed, (4, 400)) - - # poll database - table_mid = self.query_row_count(project, my_seed) - view_mid = self.query_row_count(project, my_dynamic_table) - - # refresh the materialized view - self.refresh_dynamic_table(project, my_dynamic_table) - - # poll database - table_end = self.query_row_count(project, my_seed) - view_end = self.query_row_count(project, my_dynamic_table) - - # new records were inserted in the table but didn't show up in the view until it was refreshed - assert table_start < table_mid == table_end - assert view_start == view_mid < view_end diff --git a/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py b/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py deleted file mode 100644 index a58b76f29..000000000 --- a/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py +++ /dev/null @@ -1,307 +0,0 @@ -from typing import Optional - -import pytest - -from dbt_common.contracts.config.materialization import OnConfigurationChangeOption -from dbt.tests.util import ( - assert_message_in_logs, - get_model_file, - run_dbt, - run_dbt_and_capture, - set_model_file, -) - -from dbt.adapters.snowflake.relation import SnowflakeRelation, SnowflakeRelationType -from tests.functional.adapter.dynamic_table_tests.files import ( - MY_DYNAMIC_TABLE, - MY_SEED, -) -from tests.functional.adapter.dynamic_table_tests.utils import ( - query_refresh_mode, - query_relation_type, - query_target_lag, - query_warehouse, -) - - -class SnowflakeDynamicTableChanges: - @staticmethod - def check_start_state(project, dynamic_table): - assert query_target_lag(project, dynamic_table) == "2 minutes" - assert query_warehouse(project, dynamic_table) == "DBT_TESTING" - assert query_refresh_mode(project, dynamic_table) == "INCREMENTAL" - - @staticmethod - def change_config_via_alter(project, dynamic_table): - initial_model = get_model_file(project, dynamic_table) - new_model = initial_model.replace( - "target_lag='2 minutes'", "target_lag='5 minutes'" - ) - set_model_file(project, dynamic_table, new_model) - - @staticmethod - def change_config_via_alter_downstream(project, dynamic_table): - initial_model = get_model_file(project, dynamic_table) - new_model = initial_model.replace( - "target_lag='2 minutes'", "target_lag='DOWNSTREAM'" - ) - set_model_file(project, dynamic_table, new_model) - - @staticmethod - def check_state_alter_change_is_applied(project, dynamic_table): - assert query_target_lag(project, dynamic_table) == "5 minutes" - assert query_warehouse(project, dynamic_table) == "DBT_TESTING" - - @staticmethod - def check_state_alter_change_is_applied_downstream(project, dynamic_table): - assert query_target_lag(project, dynamic_table) == "DOWNSTREAM" - assert query_warehouse(project, dynamic_table) == "DBT_TESTING" - - @staticmethod - def change_config_via_replace(project, dynamic_table): - initial_model = get_model_file(project, dynamic_table) - new_model = initial_model.replace("refresh_mode='INCREMENTAL'", "refresh_mode='FULL'") - set_model_file(project, dynamic_table, new_model) - - @staticmethod - def check_state_replace_change_is_applied(project, dynamic_table): - assert query_refresh_mode(project, dynamic_table) == "FULL" - - @staticmethod - def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: - return query_relation_type(project, relation) - - @pytest.fixture(scope="class", autouse=True) - def seeds(self): - yield {"my_seed.csv": MY_SEED} - - @pytest.fixture(scope="class", autouse=True) - def models(self): - yield {"my_dynamic_table.sql": MY_DYNAMIC_TABLE} - - @pytest.fixture(scope="class") - def my_dynamic_table(self, project) -> SnowflakeRelation: - return project.adapter.Relation.create( - identifier="my_dynamic_table", - schema=project.test_schema, - database=project.database, - type=SnowflakeRelationType.DynamicTable, - ) - - @pytest.fixture(scope="function", autouse=True) - def setup(self, project, my_dynamic_table): - # make sure the model in the data reflects the files each time - run_dbt(["seed"]) - run_dbt(["run", "--models", my_dynamic_table.identifier, "--full-refresh"]) - - # the tests touch these files, store their contents in memory - initial_model = get_model_file(project, my_dynamic_table) - - # verify the initial settings are correct in Snowflake - self.check_start_state(project, my_dynamic_table) - - yield - - # and then reset them after the test runs - set_model_file(project, my_dynamic_table, initial_model) - - # ensure clean slate each method - project.run_sql(f"drop schema if exists {project.test_schema} cascade") - - def test_full_refresh_occurs_with_changes(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - self.change_config_via_replace(project, my_dynamic_table) - _, logs = run_dbt_and_capture( - ["--debug", "run", "--models", my_dynamic_table.identifier, "--full-refresh"] - ) - - # verify the updated settings are correct in Snowflake - self.check_state_alter_change_is_applied(project, my_dynamic_table) - self.check_state_replace_change_is_applied(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', ""), False - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", logs.replace('"', "") - ) - - -class TestSnowflakeDynamicTableChangesApply(SnowflakeDynamicTableChanges): - @pytest.fixture(scope="class") - def project_config_update(self): - return {"models": {"on_configuration_change": OnConfigurationChangeOption.Apply.value}} - - def test_change_is_applied_via_alter(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_dynamic_table.name]) - - # verify the updated settings are correct in Snowflake - self.check_state_alter_change_is_applied(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', "") - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", - logs.replace('"', ""), - False, - ) - - def test_change_is_applied_via_alter_downstream(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter_downstream(project, my_dynamic_table) - _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_dynamic_table.name]) - - # verify the updated settings are correct in Snowflake - self.check_state_alter_change_is_applied_downstream(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', "") - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", - logs.replace('"', ""), - False, - ) - - @pytest.mark.skip( - "dbt-snowflake does not currently monitor any changes the trigger a full refresh" - ) - def test_change_is_applied_via_replace(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - self.change_config_via_replace(project, my_dynamic_table) - _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_dynamic_table.name]) - - # verify the updated settings are correct in Snowflake - self.check_state_alter_change_is_applied(project, my_dynamic_table) - self.check_state_replace_change_is_applied(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", logs.replace('"', "") - ) - - -class TestSnowflakeDynamicTableChangesContinue(SnowflakeDynamicTableChanges): - @pytest.fixture(scope="class") - def project_config_update(self): - return {"models": {"on_configuration_change": OnConfigurationChangeOption.Continue.value}} - - def test_change_is_not_applied_via_alter(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_dynamic_table.name]) - - # verify the updated settings are correct in Snowflake - self.check_start_state(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Configuration changes were identified and `on_configuration_change` was set" - f" to `continue` for `{my_dynamic_table}`", - logs, - ) - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', ""), False - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", - logs.replace('"', ""), - False, - ) - - def test_change_is_not_applied_via_replace(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - self.change_config_via_replace(project, my_dynamic_table) - _, logs = run_dbt_and_capture(["--debug", "run", "--models", my_dynamic_table.name]) - - # verify the updated settings are correct in Snowflake - self.check_start_state(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Configuration changes were identified and `on_configuration_change` was set" - f" to `continue` for `{my_dynamic_table}`", - logs, - ) - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', ""), False - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", - logs.replace('"', ""), - False, - ) - - -class TestSnowflakeDynamicTableChangesFailMixin(SnowflakeDynamicTableChanges): - @pytest.fixture(scope="class") - def project_config_update(self): - return {"models": {"on_configuration_change": OnConfigurationChangeOption.Fail.value}} - - def test_change_is_not_applied_via_alter(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - _, logs = run_dbt_and_capture( - ["--debug", "run", "--models", my_dynamic_table.name], expect_pass=False - ) - - # verify the updated settings are correct in Snowflake - self.check_start_state(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Configuration changes were identified and `on_configuration_change` was set" - f" to `fail` for `{my_dynamic_table}`", - logs, - ) - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', ""), False - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", - logs.replace('"', ""), - False, - ) - - def test_change_is_not_applied_via_replace(self, project, my_dynamic_table): - - # update the settings - self.change_config_via_alter(project, my_dynamic_table) - self.change_config_via_replace(project, my_dynamic_table) - _, logs = run_dbt_and_capture( - ["--debug", "run", "--models", my_dynamic_table.name], expect_pass=False - ) - - # verify the updated settings are correct in Snowflake - self.check_start_state(project, my_dynamic_table) - - # verify the settings were changed with the correct method - assert_message_in_logs( - f"Configuration changes were identified and `on_configuration_change` was set" - f" to `fail` for `{my_dynamic_table}`", - logs, - ) - assert_message_in_logs( - f"Applying ALTER to: {my_dynamic_table.render().upper()}", logs.replace('"', ""), False - ) - assert_message_in_logs( - f"Applying REPLACE to: {my_dynamic_table.render().upper()}", - logs.replace('"', ""), - False, - ) diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py deleted file mode 100644 index d72b231c9..000000000 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ /dev/null @@ -1,53 +0,0 @@ -from typing import Optional - -import agate -from dbt.tests.util import get_connection - -from dbt.adapters.snowflake.relation import SnowflakeRelation - - -def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: - sql = f""" - select - case - when table_type = 'BASE TABLE' and is_dynamic = 'YES' then 'dynamic_table' - when table_type = 'BASE TABLE' then 'table' - when table_type = 'VIEW' then 'view' - when table_type = 'EXTERNAL TABLE' then 'external_table' - end as relation_type - from information_schema.tables - where table_name like '{relation.identifier.upper()}' - and table_schema like '{relation.schema.upper()}' - and table_catalog like '{relation.database.upper()}' - """ - results = project.run_sql(sql, fetch="one") - if results is None or len(results) == 0: - return None - elif len(results) > 1: - raise ValueError(f"More than one instance of {relation.name} found!") - else: - return results[0].lower() - - -def query_target_lag(project, dynamic_table: SnowflakeRelation) -> Optional[str]: - config = describe_dynamic_table(project, dynamic_table) - return config.get("target_lag") - - -def query_warehouse(project, dynamic_table: SnowflakeRelation) -> Optional[str]: - config = describe_dynamic_table(project, dynamic_table) - return config.get("warehouse") - - -def query_refresh_mode(project, dynamic_table: SnowflakeRelation) -> Optional[str]: - config = describe_dynamic_table(project, dynamic_table) - return config.get("refresh_mode") - - -def describe_dynamic_table(project, dynamic_table: SnowflakeRelation) -> agate.Row: - with get_connection(project.adapter): - macro_results = project.adapter.execute_macro( - "snowflake__describe_dynamic_table", kwargs={"relation": dynamic_table} - ) - config = macro_results["dynamic_table"] - return config.rows[0] diff --git a/tests/functional/relation_tests/__init__.py b/tests/functional/relation_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/relation_tests/dynamic_table_tests/__init__.py b/tests/functional/relation_tests/dynamic_table_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/relation_tests/dynamic_table_tests/models.py b/tests/functional/relation_tests/dynamic_table_tests/models.py new file mode 100644 index 000000000..5e46bed53 --- /dev/null +++ b/tests/functional/relation_tests/dynamic_table_tests/models.py @@ -0,0 +1,50 @@ +SEED = """ +id,value +1,100 +2,200 +3,300 +""".strip() + + +DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE_DOWNSTREAM = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='DOWNSTREAM', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE_ALTER = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='5 minutes', + refresh_mode='INCREMENTAL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_TABLE_REPLACE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='FULL', +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py new file mode 100644 index 000000000..2406e1c14 --- /dev/null +++ b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py @@ -0,0 +1,30 @@ +import pytest + +from dbt.tests.util import run_dbt + +from tests.functional.relation_tests.dynamic_table_tests import models +from tests.functional.utils import query_relation_type + + +class TestBasic: + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_dynamic_table.sql": models.DYNAMIC_TABLE, + "my_dynamic_table_downstream.sql": models.DYNAMIC_TABLE_DOWNSTREAM, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + def test_dynamic_table_full_refresh(self, project): + run_dbt(["run", "--full-refresh"]) + assert query_relation_type(project, "my_dynamic_table") == "dynamic_table" + assert query_relation_type(project, "my_dynamic_table_downstream") == "dynamic_table" diff --git a/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py b/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py new file mode 100644 index 000000000..3c4f65a87 --- /dev/null +++ b/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py @@ -0,0 +1,103 @@ +import pytest + +from dbt.tests.util import run_dbt + +from tests.functional.relation_tests.dynamic_table_tests import models +from tests.functional.utils import describe_dynamic_table, update_model + + +class Changes: + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + yield {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "dynamic_table_alter.sql": models.DYNAMIC_TABLE, + "dynamic_table_replace.sql": models.DYNAMIC_TABLE, + } + + @pytest.fixture(scope="function", autouse=True) + def setup_class(self, project): + run_dbt(["seed"]) + yield + project.run_sql(f"drop schema if exists {project.test_schema} cascade") + + @pytest.fixture(scope="function", autouse=True) + def setup_method(self, project, setup_class): + # make sure the model in the data reflects the files each time + run_dbt(["run", "--full-refresh"]) + self.assert_changes_are_not_applied(project) + + update_model(project, "dynamic_table_alter", models.DYNAMIC_TABLE_ALTER) + update_model(project, "dynamic_table_replace", models.DYNAMIC_TABLE_REPLACE) + + yield + + update_model(project, "dynamic_table_alter", models.DYNAMIC_TABLE) + update_model(project, "dynamic_table_replace", models.DYNAMIC_TABLE) + + @staticmethod + def assert_changes_are_applied(project): + altered = describe_dynamic_table(project, "dynamic_table_alter") + assert altered.snowflake_warehouse == "DBT_TESTING" + assert altered.target_lag == "5 minutes" # this updated + assert altered.refresh_mode == "INCREMENTAL" + + replaced = describe_dynamic_table(project, "dynamic_table_replace") + assert replaced.snowflake_warehouse == "DBT_TESTING" + assert replaced.target_lag == "2 minutes" + assert replaced.refresh_mode == "FULL" # this updated + + @staticmethod + def assert_changes_are_not_applied(project): + altered = describe_dynamic_table(project, "dynamic_table_alter") + assert altered.snowflake_warehouse == "DBT_TESTING" + assert altered.target_lag == "2 minutes" # this would have updated, but didn't + assert altered.refresh_mode == "INCREMENTAL" + + replaced = describe_dynamic_table(project, "dynamic_table_replace") + assert replaced.snowflake_warehouse == "DBT_TESTING" + assert replaced.target_lag == "2 minutes" + assert replaced.refresh_mode == "INCREMENTAL" # this would have updated, but didn't + + def test_full_refresh_is_always_successful(self, project): + # this always passes and always changes the configuration, regardless of on_configuration_change + # and regardless of whether the changes require a replace versus an alter + run_dbt(["run", "--full-refresh"]) + self.assert_changes_are_applied(project) + + +class TestChangesApply(Changes): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": "apply"}} + + def test_changes_are_applied(self, project): + # this passes and changes the configuration + run_dbt(["run"]) + self.assert_changes_are_applied(project) + + +class TestChangesContinue(Changes): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": "continue"}} + + def test_changes_are_not_applied(self, project): + # this passes but does not change the configuration + run_dbt(["run"]) + self.assert_changes_are_not_applied(project) + + +class TestChangesFail(Changes): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"on_configuration_change": "fail"}} + + def test_changes_are_not_applied(self, project): + # this fails and does not change the configuration + run_dbt(["run"], expect_pass=False) + self.assert_changes_are_not_applied(project) diff --git a/tests/functional/adapter/dynamic_table_tests/files.py b/tests/functional/relation_tests/models.py similarity index 78% rename from tests/functional/adapter/dynamic_table_tests/files.py rename to tests/functional/relation_tests/models.py index ef8d2bf1f..6fe066313 100644 --- a/tests/functional/adapter/dynamic_table_tests/files.py +++ b/tests/functional/relation_tests/models.py @@ -1,4 +1,4 @@ -MY_SEED = """ +SEED = """ id,value 1,100 2,200 @@ -6,7 +6,7 @@ """.strip() -MY_TABLE = """ +TABLE = """ {{ config( materialized='table', ) }} @@ -14,7 +14,7 @@ """ -MY_VIEW = """ +VIEW = """ {{ config( materialized='view', ) }} @@ -22,11 +22,11 @@ """ -MY_DYNAMIC_TABLE = """ +DYNAMIC_TABLE = """ {{ config( materialized='dynamic_table', snowflake_warehouse='DBT_TESTING', - target_lag='2 minutes', + target_lag='1 minute', refresh_mode='INCREMENTAL', ) }} select * from {{ ref('my_seed') }} diff --git a/tests/functional/relation_tests/test_relation_type_change.py b/tests/functional/relation_tests/test_relation_type_change.py new file mode 100644 index 000000000..1246b0791 --- /dev/null +++ b/tests/functional/relation_tests/test_relation_type_change.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass +from itertools import product + +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.relation_tests import models +from tests.functional.utils import query_relation_type, update_model + + +@dataclass +class Model: + model: str + relation_type: str + + @property + def name(self) -> str: + return f"{self.relation_type}" + + +@dataclass +class Scenario: + initial: Model + final: Model + + @property + def name(self) -> str: + return f"REPLACE_{self.initial.name}__WITH_{self.final.name}" + + @property + def error_message(self) -> str: + return f"Failed when migrating from: {self.initial.name} to: {self.final.name}" + + +relations = [ + Model(models.VIEW, "view"), + Model(models.TABLE, "table"), + Model(models.DYNAMIC_TABLE, "dynamic_table"), +] +scenarios = [Scenario(*scenario) for scenario in product(relations, relations)] + + +class TestRelationTypeChange: + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield {f"{scenario.name}.sql": scenario.initial.model for scenario in scenarios} + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + for scenario in scenarios: + update_model(project, scenario.name, scenario.final.model) + run_dbt(["run"]) + + @pytest.mark.parametrize("scenario", scenarios, ids=[scenario.name for scenario in scenarios]) + def test_replace(self, project, scenario): + relation_type = query_relation_type(project, scenario.name) + assert relation_type == scenario.final.relation_type, scenario.error_message diff --git a/tests/functional/utils.py b/tests/functional/utils.py new file mode 100644 index 000000000..d185e8d2b --- /dev/null +++ b/tests/functional/utils.py @@ -0,0 +1,78 @@ +from typing import Any, Dict, Optional + +from dbt.tests.util import ( + get_connection, + get_model_file, + relation_from_name, + set_model_file, +) + +from dbt.adapters.snowflake.relation_configs import SnowflakeDynamicTableConfig + + +def query_relation_type(project, name: str) -> Optional[str]: + relation = relation_from_name(project.adapter, name) + sql = f""" + select + case table_type + when 'BASE TABLE' then iff(is_dynamic = 'YES', 'dynamic_table', 'table') + when 'VIEW' then 'view' + when 'EXTERNAL TABLE' then 'external_table' + end as relation_type + from information_schema.tables + where table_name like '{relation.identifier.upper()}' + and table_schema like '{relation.schema.upper()}' + and table_catalog like '{relation.database.upper()}' + """ + results = project.run_sql(sql, fetch="all") + + assert len(results) > 0, f"Relation {relation} not found" + assert len(results) == 1, f"Multiple relations found" + + return results[0][0].lower() + + +def query_row_count(project, name: str) -> int: + relation = relation_from_name(project.adapter, name) + sql = f"select count(*) from {relation}" + return project.run_sql(sql, fetch="one")[0] + + +def insert_record(project, name: str, record: Dict[str, Any]): + relation = relation_from_name(project.adapter, name) + column_names = ", ".join(record.keys()) + values = ", ".join( + [f"'{value}'" if isinstance(value, str) else f"{value}" for value in record.values()] + ) + sql = f"insert into {relation} ({column_names}) values ({values})" + project.run_sql(sql) + + +def update_model(project, name: str, model: str) -> str: + relation = relation_from_name(project.adapter, name) + original_model = get_model_file(project, relation) + set_model_file(project, relation, model) + return original_model + + +def describe_dynamic_table(project, name: str) -> Optional[SnowflakeDynamicTableConfig]: + macro = "snowflake__describe_dynamic_table" + dynamic_table = relation_from_name(project.adapter, name) + kwargs = {"relation": dynamic_table} + with get_connection(project.adapter): + results = project.adapter.execute_macro(macro, kwargs=kwargs) + + assert len(results["dynamic_table"].rows) > 0, f"Dynamic table {dynamic_table} not found" + found = len(results["dynamic_table"].rows) + names = ", ".join([table.get("name") for table in results["dynamic_table"].rows]) + assert found == 1, f"Multiple dynamic tables found: {names}" + + return SnowflakeDynamicTableConfig.from_relation_results(results) + + +def refresh_dynamic_table(project, name: str) -> None: + macro = "snowflake__refresh_dynamic_table" + dynamic_table = relation_from_name(project.adapter, name) + kwargs = {"relation": dynamic_table} + with get_connection(project.adapter): + project.adapter.execute_macro(macro, kwargs=kwargs) From 583ec5eb2686720b5715b0121c4c7d79a3e319d3 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 23 Sep 2024 22:57:17 +0100 Subject: [PATCH 08/41] Add required 'begin' config for testing microbatch models (#1189) --- tests/functional/adapter/test_incremental_microbatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_incremental_microbatch.py b/tests/functional/adapter/test_incremental_microbatch.py index bbb57f96c..f228c370c 100644 --- a/tests/functional/adapter/test_incremental_microbatch.py +++ b/tests/functional/adapter/test_incremental_microbatch.py @@ -6,7 +6,7 @@ # No requirement for a unique_id for snowflake microbatch! _microbatch_model_no_unique_id_sql = """ -{{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day') }} +{{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model') }} """ From d7632ebb2b36daa108e9ad2b2374d4a4c7855145 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:05:02 -0700 Subject: [PATCH 09/41] Add logic to enable iceberg incremental tables. (#1190) * Add logic to enable iceberg incremental tables. * Add changelog. * Standardize existing_relation as name of existing model. * Improve error message for table formats. * Update error message and add round of tests. * Add more comprehensive tests for before/after. * Update identifier param in incremental materialization. * Import Mike's revision on the relation type change test. * Try adding iceberg incremental model scenarios. * Disable dynamic tests. * Refine booleans for more restricting when incremental models are built. * Update tests to reflect realities on the database. * Add additional metadata. * Update boolean logic for faster runs overall. * Last bit of cleanup per code review * Syntax error from lack of iterable. --- .../unreleased/Features-20240923-203204.yaml | 6 + .../macros/materializations/incremental.sql | 23 +++- .../macros/materializations/table.sql | 8 +- .../iceberg/test_incremental_models.py | 126 ++++++++++++++++++ tests/functional/relation_tests/models.py | 34 +++++ .../test_relation_type_change.py | 75 +++++++++-- 6 files changed, 258 insertions(+), 14 deletions(-) create mode 100644 .changes/unreleased/Features-20240923-203204.yaml create mode 100644 tests/functional/iceberg/test_incremental_models.py diff --git a/.changes/unreleased/Features-20240923-203204.yaml b/.changes/unreleased/Features-20240923-203204.yaml new file mode 100644 index 000000000..eaca4906b --- /dev/null +++ b/.changes/unreleased/Features-20240923-203204.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add Iceberg format Incremental Models +time: 2024-09-23T20:32:04.783741-07:00 +custom: + Author: versusfacit + Issue: "321" diff --git a/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt/include/snowflake/macros/materializations/incremental.sql index 9172c061e..12645cd11 100644 --- a/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/dbt/include/snowflake/macros/materializations/incremental.sql @@ -58,12 +58,21 @@ {% materialization incremental, adapter='snowflake', supported_languages=['sql', 'python'] -%} - {% set original_query_tag = set_query_tag() %} {#-- Set vars --#} {%- set full_refresh_mode = (should_full_refresh()) -%} {%- set language = model['language'] -%} - {% set target_relation = this %} + + {%- set identifier = this.name -%} + + {%- set target_relation = api.Relation.create( + identifier=identifier, + schema=schema, + database=database, + type='table', + table_format=config.get('table_format', 'default') + ) -%} + {% set existing_relation = load_relation(this) %} {#-- The temp relation will be a view (faster) or temp table, depending on upsert/merge strategy --#} @@ -90,11 +99,21 @@ {%- call statement('main', language=language) -%} {{ create_table_as(False, target_relation, compiled_code, language) }} {%- endcall -%} + {% elif full_refresh_mode %} + {% if target_relation.needs_to_drop(existing_relation) %} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} {%- call statement('main', language=language) -%} {{ create_table_as(False, target_relation, compiled_code, language) }} {%- endcall -%} + {% elif target_relation.table_format != existing_relation.table_format %} + {% do exceptions.raise_compiler_error( + "Unable to alter incremental model `" ~ target_relation.identifier ~ "` to '" ~ target_relation.table_format ~ " table format due to Snowflake limitation. Please execute with --full-refresh to drop the table and recreate in new table format.'" + ) + %} + {% else %} {#-- Create the temp relation, either as a view or as a temp table --#} {% if tmp_relation_type == 'view' %} diff --git a/dbt/include/snowflake/macros/materializations/table.sql b/dbt/include/snowflake/macros/materializations/table.sql index 9ee8a0b12..995757b6b 100644 --- a/dbt/include/snowflake/macros/materializations/table.sql +++ b/dbt/include/snowflake/macros/materializations/table.sql @@ -7,7 +7,7 @@ {% set grant_config = config.get('grants') %} - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set existing_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create( identifier=identifier, schema=schema, @@ -18,8 +18,8 @@ {{ run_hooks(pre_hooks) }} - {% if target_relation.needs_to_drop(old_relation) %} - {{ drop_relation_if_exists(old_relation) }} + {% if target_relation.needs_to_drop(existing_relation) %} + {{ drop_relation_if_exists(existing_relation) }} {% endif %} {% call statement('main', language=language) -%} @@ -28,7 +28,7 @@ {{ run_hooks(post_hooks) }} - {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %} + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} {% do persist_docs(target_relation, model) %} diff --git a/tests/functional/iceberg/test_incremental_models.py b/tests/functional/iceberg/test_incremental_models.py new file mode 100644 index 000000000..a02d9ffed --- /dev/null +++ b/tests/functional/iceberg/test_incremental_models.py @@ -0,0 +1,126 @@ +import pytest + +from pathlib import Path + +from dbt.tests.util import run_dbt, run_dbt_and_capture + + +_SEED_INCREMENTAL_STRATEGIES = """ +world_id,world_name,boss +1,Yoshi's Island,Iggy +2,Donut Plains,Morton +3,Vanilla Dome,Lemmy +4,Cookie Mountain,Temmy +5,Forest of Illusion,Roy +""".strip() + +_MODEL_BASIC_TABLE_MODEL = """ +{{ + config( + materialized = "table", + ) +}} +select * from {{ ref('seed') }} +""" + +_MODEL_INCREMENTAL_ICEBERG_BASE = """ +{{{{ + config( + materialized='incremental', + table_format='iceberg', + incremental_strategy='{strategy}', + unique_key="world_id", + external_volume = "s3_iceberg_snow", + ) +}}}} +select * from {{{{ ref('upstream_table') }}}} + +{{% if is_incremental() %}} +where world_id > 2 +{{% endif %}} +""" + +_MODEL_INCREMENTAL_ICEBERG_APPEND = _MODEL_INCREMENTAL_ICEBERG_BASE.format(strategy="append") +_MODEL_INCREMENTAL_ICEBERG_MERGE = _MODEL_INCREMENTAL_ICEBERG_BASE.format(strategy="merge") +_MODEL_INCREMENTAL_ICEBERG_DELETE_INSERT = _MODEL_INCREMENTAL_ICEBERG_BASE.format( + strategy="delete+insert" +) + + +_QUERY_UPDATE_UPSTREAM_TABLE = """ +UPDATE {database}.{schema}.upstream_table set world_name = 'Twin Bridges', boss = 'Ludwig' where world_id = 4; +""" + +_QUERY_UPDATE_UPSTREAM_TABLE_NO_EFFECT = """ +UPDATE {database}.{schema}.upstream_table set world_name = 'Doughnut Plains' where world_id = 2; +""" + + +class TestIcebergIncrementalStrategies: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEED_INCREMENTAL_STRATEGIES, + } + + @pytest.fixture(scope="function", autouse=True) + def setup_class(self, project): + run_dbt(["seed"]) + yield + + @pytest.fixture(scope="class") + def models(self): + return { + "upstream_table.sql": _MODEL_BASIC_TABLE_MODEL, + "append.sql": _MODEL_INCREMENTAL_ICEBERG_APPEND, + "merge.sql": _MODEL_INCREMENTAL_ICEBERG_MERGE, + "delete_insert.sql": _MODEL_INCREMENTAL_ICEBERG_DELETE_INSERT, + } + + def test_incremental_strategies_build(self, project, setup_class): + run_results = run_dbt() + assert len(run_results) == 4 + + def __check_correct_operations(self, model_name, /, rows_affected, status="SUCCESS"): + run_results = run_dbt( + ["show", "--inline", f"select * from {{{{ ref('{model_name}') }}}} where world_id = 4"] + ) + assert run_results[0].adapter_response["rows_affected"] == rows_affected + assert run_results[0].adapter_response["code"] == status + + if model_name != "append": + run_results, stdout = run_dbt_and_capture( + [ + "show", + "--inline", + f"select * from {{{{ ref('{model_name}') }}}} where world_id = 2", + ] + ) + run_results[0].adapter_response["rows_affected"] == 1 + assert "Doughnut" not in stdout + + def test_incremental_strategies_with_update(self, project, setup_class): + run_results = run_dbt() + assert len(run_results) == 4 + + project.run_sql( + _QUERY_UPDATE_UPSTREAM_TABLE.format( + database=project.database, schema=project.test_schema + ) + ) + project.run_sql( + _QUERY_UPDATE_UPSTREAM_TABLE_NO_EFFECT.format( + database=project.database, schema=project.test_schema + ) + ) + + run_results = run_dbt(["run", "-s", "append", "merge", "delete_insert"]) + assert len(run_results) == 3 + + self.__check_correct_operations("append", rows_affected=3) + self.__check_correct_operations("merge", rows_affected=1) + self.__check_correct_operations("delete_insert", rows_affected=1) diff --git a/tests/functional/relation_tests/models.py b/tests/functional/relation_tests/models.py index 6fe066313..63dfff045 100644 --- a/tests/functional/relation_tests/models.py +++ b/tests/functional/relation_tests/models.py @@ -31,3 +31,37 @@ ) }} select * from {{ ref('my_seed') }} """ + + +DYNAMIC_ICEBERG_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" + +ICEBERG_TABLE = """ +{{ config( + materialized='table', + table_format="iceberg", + external_volume="s3_iceberg_snow", +) }} +select * from {{ ref('my_seed') }} +""" + +ICEBERG_INCREMENTAL_TABLE = """ +{{ config( + materialized='incremental', + table_format='iceberg', + incremental_strategy='append', + unique_key="id", + external_volume = "s3_iceberg_snow", +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/tests/functional/relation_tests/test_relation_type_change.py b/tests/functional/relation_tests/test_relation_type_change.py index 1246b0791..c2886ad04 100644 --- a/tests/functional/relation_tests/test_relation_type_change.py +++ b/tests/functional/relation_tests/test_relation_type_change.py @@ -1,21 +1,27 @@ from dataclasses import dataclass from itertools import product +from typing import Optional from dbt.tests.util import run_dbt import pytest from tests.functional.relation_tests import models -from tests.functional.utils import query_relation_type, update_model +from tests.functional.utils import describe_dynamic_table, query_relation_type, update_model @dataclass class Model: model: str relation_type: str + table_format: Optional[str] = None + incremental: Optional[bool] = None @property def name(self) -> str: - return f"{self.relation_type}" + name = f"{self.relation_type}" + if self.table_format: + name += f"_{self.table_format}" + return name @dataclass @@ -34,31 +40,84 @@ def error_message(self) -> str: relations = [ Model(models.VIEW, "view"), - Model(models.TABLE, "table"), - Model(models.DYNAMIC_TABLE, "dynamic_table"), + Model(models.TABLE, "table", "default"), + # to be activated upon merge of dynamic table support PR + # Model(models.DYNAMIC_TABLE, "dynamic_table", "default"), + # Model(models.DYNAMIC_ICEBERG_TABLE, "dynamic_table", "iceberg"), + Model(models.ICEBERG_TABLE, "table", "iceberg"), + Model(models.ICEBERG_INCREMENTAL_TABLE, "table", "iceberg", incremental=True), ] scenarios = [Scenario(*scenario) for scenario in product(relations, relations)] class TestRelationTypeChange: + @staticmethod + def include(scenario) -> bool: + return ( + scenario.initial.table_format != "iceberg" and scenario.final.table_format != "iceberg" + ) + @pytest.fixture(scope="class", autouse=True) def seeds(self): return {"my_seed.csv": models.SEED} @pytest.fixture(scope="class", autouse=True) def models(self): - yield {f"{scenario.name}.sql": scenario.initial.model for scenario in scenarios} + yield { + f"{scenario.name}.sql": scenario.initial.model + for scenario in scenarios + if self.include(scenario) + } @pytest.fixture(scope="class", autouse=True) def setup(self, project): run_dbt(["seed"]) run_dbt(["run"]) for scenario in scenarios: - update_model(project, scenario.name, scenario.final.model) + if self.include(scenario): + update_model(project, scenario.name, scenario.final.model) run_dbt(["run"]) @pytest.mark.parametrize("scenario", scenarios, ids=[scenario.name for scenario in scenarios]) def test_replace(self, project, scenario): - relation_type = query_relation_type(project, scenario.name) - assert relation_type == scenario.final.relation_type, scenario.error_message + if self.include(scenario): + relation_type = query_relation_type(project, scenario.name) + assert relation_type == scenario.final.relation_type, scenario.error_message + if relation_type == "dynamic_table": + dynamic_table = describe_dynamic_table(project, scenario.name) + assert dynamic_table.catalog.table_format == scenario.final.table_format + else: + pytest.skip() + + +""" +Upon adding the logic needed for seamless transitions to and from incremental models without data loss, we can coalesce these test cases. +""" + + +class TestRelationTypeChangeIcebergOn(TestRelationTypeChange): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @staticmethod + def include(scenario) -> bool: + return any( + ( + # scenario 1: Everything that doesn't include incremental relations on Iceberg + ( + ( + scenario.initial.table_format == "iceberg" + or scenario.final.table_format == "iceberg" + ) + and not scenario.initial.incremental + and not scenario.final.incremental + ), + # scenario 2: Iceberg Incremental swaps allowed + ( + scenario.initial.table_format == "iceberg" + and scenario.final.table_format == "iceberg" + ), + ) + ) From 0ae7479227bba2da53113aac27758183a00d5ab6 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:50:22 -0700 Subject: [PATCH 10/41] update dbt-common dependency to 1.10 and dbt-adapters to 1.7 (#1193) Co-authored-by: Mila Page <67295367+VersusFacit@users.noreply.github.com> --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 210c309b1..f542b9fcb 100644 --- a/setup.py +++ b/setup.py @@ -57,8 +57,8 @@ def _plugin_version() -> str: packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=1.3.0,<2.0", - "dbt-adapters>=1.3.1,<2.0", + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.7,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 423111f5c73b3362778ee0abbaef2a1fcd09fb19 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:34:06 -0700 Subject: [PATCH 11/41] Adap 321/add iceberg incremental models (#1194) * Update expected rows to reflect what the append strategy actually does * Empty commit to shift the test schema of the test off the bugged iceberg model * More robust solution to avoid time-based model conflicts. * prints on ci to see what's happening. * Remove the superfluous test that is causing metadata conflicts for append. --- .../functional/iceberg/test_incremental_models.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/functional/iceberg/test_incremental_models.py b/tests/functional/iceberg/test_incremental_models.py index a02d9ffed..f8f1d6b89 100644 --- a/tests/functional/iceberg/test_incremental_models.py +++ b/tests/functional/iceberg/test_incremental_models.py @@ -1,4 +1,5 @@ import pytest +import time from pathlib import Path @@ -57,6 +58,8 @@ class TestIcebergIncrementalStrategies: + append: str = f"append_{hash(time.time())}" + @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"enable_iceberg_materializations": True}} @@ -76,15 +79,11 @@ def setup_class(self, project): def models(self): return { "upstream_table.sql": _MODEL_BASIC_TABLE_MODEL, - "append.sql": _MODEL_INCREMENTAL_ICEBERG_APPEND, + f"{self.append}.sql": _MODEL_INCREMENTAL_ICEBERG_APPEND, "merge.sql": _MODEL_INCREMENTAL_ICEBERG_MERGE, "delete_insert.sql": _MODEL_INCREMENTAL_ICEBERG_DELETE_INSERT, } - def test_incremental_strategies_build(self, project, setup_class): - run_results = run_dbt() - assert len(run_results) == 4 - def __check_correct_operations(self, model_name, /, rows_affected, status="SUCCESS"): run_results = run_dbt( ["show", "--inline", f"select * from {{{{ ref('{model_name}') }}}} where world_id = 4"] @@ -92,7 +91,7 @@ def __check_correct_operations(self, model_name, /, rows_affected, status="SUCCE assert run_results[0].adapter_response["rows_affected"] == rows_affected assert run_results[0].adapter_response["code"] == status - if model_name != "append": + if "append" not in model_name: run_results, stdout = run_dbt_and_capture( [ "show", @@ -118,9 +117,9 @@ def test_incremental_strategies_with_update(self, project, setup_class): ) ) - run_results = run_dbt(["run", "-s", "append", "merge", "delete_insert"]) + run_results = run_dbt(["run", "-s", self.append, "merge", "delete_insert"]) assert len(run_results) == 3 - self.__check_correct_operations("append", rows_affected=3) + self.__check_correct_operations(self.append, rows_affected=2) self.__check_correct_operations("merge", rows_affected=1) self.__check_correct_operations("delete_insert", rows_affected=1) From 0521395a303bf654acfa38cc0cd927d8f7ec4485 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:52:04 -0400 Subject: [PATCH 12/41] Add support for Iceberg table format in Dynamic Tables (#1183) * add support for iceberg dynamic tables * remove is_dynamic-related guards as that is ga now * simplify dynamic table testing * add iceberg dynamic tables to existing dynamic table tests * add standard incremental tables into the relation swap scenarios * account for the fact that snowflake does not support renaming iceberg relations * account for all scenarios when swapping relation types, including those which currently require a full refresh * make it clearer which scenarios are included in each run and why by pulling the criteria into one function --------- Co-authored-by: Mila Page <67295367+VersusFacit@users.noreply.github.com> --- .../unreleased/Features-20240917-100505.yaml | 6 + dbt/adapters/snowflake/relation.py | 15 +++ .../snowflake/relation_configs/__init__.py | 6 +- .../snowflake/relation_configs/catalog.py | 123 ++++++++++++++++++ .../relation_configs/dynamic_table.py | 24 +++- .../snowflake/relation_configs/formats.py | 5 + .../macros/relations/dynamic_table/create.sql | 79 ++++++++++- .../relations/dynamic_table/describe.sql | 39 +++++- .../relations/dynamic_table/replace.sql | 78 ++++++++++- .../snowflake/macros/utils/optional.sql | 14 ++ .../dynamic_table_tests/models.py | 48 ++++++- .../dynamic_table_tests/test_basic.py | 20 ++- .../test_configuration_changes.py | 84 +++++++++++- tests/functional/relation_tests/models.py | 12 +- .../test_relation_type_change.py | 106 ++++++++++----- 15 files changed, 593 insertions(+), 66 deletions(-) create mode 100644 .changes/unreleased/Features-20240917-100505.yaml create mode 100644 dbt/adapters/snowflake/relation_configs/catalog.py create mode 100644 dbt/include/snowflake/macros/utils/optional.sql diff --git a/.changes/unreleased/Features-20240917-100505.yaml b/.changes/unreleased/Features-20240917-100505.yaml new file mode 100644 index 000000000..22cabc904 --- /dev/null +++ b/.changes/unreleased/Features-20240917-100505.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for Iceberg table format in Dynamic Tables +time: 2024-09-17T10:05:05.609859-04:00 +custom: + Author: mikealfare + Issue: "1183" diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 224b2b75e..b6924b9b3 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -17,6 +17,7 @@ from dbt_common.events.functions import fire_event, warn_or_error from dbt.adapters.snowflake.relation_configs import ( + SnowflakeCatalogConfigChange, SnowflakeDynamicTableConfig, SnowflakeDynamicTableConfigChangeset, SnowflakeDynamicTableRefreshModeConfigChange, @@ -114,6 +115,12 @@ def dynamic_table_config_changeset( context=new_dynamic_table.refresh_mode, ) + if new_dynamic_table.catalog != existing_dynamic_table.catalog: + config_change_collection.catalog = SnowflakeCatalogConfigChange( + action=RelationConfigChangeAction.create, + context=new_dynamic_table.catalog, + ) + if config_change_collection.has_changes: return config_change_collection return None @@ -132,6 +139,14 @@ def as_case_sensitive(self) -> "SnowflakeRelation": return self.replace_path(**path_part_map) + @property + def can_be_renamed(self) -> bool: + """ + Standard tables and dynamic tables can be renamed, but Snowflake does not support renaming iceberg relations. + The iceberg standard does support renaming, so this may change in the future. + """ + return self.type in self.renameable_relations and not self.is_iceberg_format + def get_ddl_prefix_for_create(self, config: RelationConfig, temporary: bool) -> str: """ This macro renders the appropriate DDL prefix during the create_table_as diff --git a/dbt/adapters/snowflake/relation_configs/__init__.py b/dbt/adapters/snowflake/relation_configs/__init__.py index 61941ab50..fec9d8a54 100644 --- a/dbt/adapters/snowflake/relation_configs/__init__.py +++ b/dbt/adapters/snowflake/relation_configs/__init__.py @@ -1,3 +1,7 @@ +from dbt.adapters.snowflake.relation_configs.catalog import ( + SnowflakeCatalogConfig, + SnowflakeCatalogConfigChange, +) from dbt.adapters.snowflake.relation_configs.dynamic_table import ( SnowflakeDynamicTableConfig, SnowflakeDynamicTableConfigChangeset, @@ -5,9 +9,9 @@ SnowflakeDynamicTableWarehouseConfigChange, SnowflakeDynamicTableTargetLagConfigChange, ) +from dbt.adapters.snowflake.relation_configs.formats import TableFormat from dbt.adapters.snowflake.relation_configs.policies import ( SnowflakeIncludePolicy, SnowflakeQuotePolicy, SnowflakeRelationType, ) -from dbt.adapters.snowflake.relation_configs.formats import TableFormat diff --git a/dbt/adapters/snowflake/relation_configs/catalog.py b/dbt/adapters/snowflake/relation_configs/catalog.py new file mode 100644 index 000000000..09e338635 --- /dev/null +++ b/dbt/adapters/snowflake/relation_configs/catalog.py @@ -0,0 +1,123 @@ +from dataclasses import dataclass +from typing import Any, Dict, Optional, TYPE_CHECKING, Set + +if TYPE_CHECKING: + import agate + +from dbt.adapters.relation_configs import ( + RelationConfigChange, + RelationResults, + RelationConfigValidationMixin, + RelationConfigValidationRule, +) +from dbt.adapters.contracts.relation import RelationConfig +from dbt_common.exceptions import DbtConfigError +from typing_extensions import Self + +from dbt.adapters.snowflake.relation_configs.base import SnowflakeRelationConfigBase +from dbt.adapters.snowflake.relation_configs.formats import TableFormat + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeCatalogConfig(SnowflakeRelationConfigBase, RelationConfigValidationMixin): + """ + This config follow the specs found here: + https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table + https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table + + The following parameters are configurable by dbt: + - table_format: format for interfacing with the table, e.g. default, iceberg + - external_volume: name of the external volume in Snowflake + - base_location: the directory within the external volume that contains the data + *Note*: This directory can’t be changed after you create a table. + + The following parameters are not currently configurable by dbt: + - name: snowflake + """ + + table_format: Optional[TableFormat] = TableFormat.default() + name: Optional[str] = "SNOWFLAKE" + external_volume: Optional[str] = None + base_location: Optional[str] = None + + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + return { + RelationConfigValidationRule( + (self.table_format == "default") + or (self.table_format == "iceberg" and self.base_location is not None), + DbtConfigError("Please provide a `base_location` when using iceberg"), + ), + RelationConfigValidationRule( + (self.table_format == "default") + or (self.table_format == "iceberg" and self.name == "SNOWFLAKE"), + DbtConfigError( + "Only Snowflake catalogs are currently supported when using iceberg" + ), + ), + } + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any]) -> Self: + kwargs_dict = { + "name": config_dict.get("name"), + "external_volume": config_dict.get("external_volume"), + "base_location": config_dict.get("base_location"), + } + if table_format := config_dict.get("table_format"): + kwargs_dict["table_format"] = TableFormat(table_format) + return super().from_dict(kwargs_dict) + + @classmethod + def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: + + if relation_config.config.extra.get("table_format") is None: + return {} + + config_dict = { + "table_format": relation_config.config.extra.get("table_format"), + "name": "SNOWFLAKE", # this is not currently configurable + } + + if external_volume := relation_config.config.extra.get("external_volume"): + config_dict["external_volume"] = external_volume + + if base_location := relation_config.config.extra.get("base_location_subpath"): + config_dict["base_location"] = base_location + + return config_dict + + @classmethod + def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]: + # this try block can be removed once enable_iceberg_materializations is retired + try: + catalog_results: "agate.Table" = relation_results["catalog"] + except KeyError: + # this happens when `enable_iceberg_materializations` is turned off + return {} + + if len(catalog_results) == 0: + # this happens when the dynamic table is a standard dynamic table (e.g. not iceberg) + return {} + + # for now, if we get catalog results, it's because this is an iceberg table + # this is because we only run `show iceberg tables` to get catalog metadata + # this will need to be updated once this is in `show objects` + catalog: "agate.Row" = catalog_results.rows[0] + config_dict = { + "table_format": "iceberg", + "name": catalog.get("catalog_name"), + "external_volume": catalog.get("external_volume_name"), + "base_location": catalog.get("base_location"), + } + + return config_dict + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class SnowflakeCatalogConfigChange(RelationConfigChange): + context: Optional[SnowflakeCatalogConfig] = None + + @property + def requires_full_refresh(self) -> bool: + return True diff --git a/dbt/adapters/snowflake/relation_configs/dynamic_table.py b/dbt/adapters/snowflake/relation_configs/dynamic_table.py index 2e227d3a4..7361df80a 100644 --- a/dbt/adapters/snowflake/relation_configs/dynamic_table.py +++ b/dbt/adapters/snowflake/relation_configs/dynamic_table.py @@ -8,6 +8,11 @@ from typing_extensions import Self from dbt.adapters.snowflake.relation_configs.base import SnowflakeRelationConfigBase +from dbt.adapters.snowflake.relation_configs.catalog import ( + SnowflakeCatalogConfig, + SnowflakeCatalogConfigChange, +) + if TYPE_CHECKING: import agate @@ -55,11 +60,12 @@ class SnowflakeDynamicTableConfig(SnowflakeRelationConfigBase): query: str target_lag: str snowflake_warehouse: str + catalog: SnowflakeCatalogConfig refresh_mode: Optional[RefreshMode] = RefreshMode.default() initialize: Optional[Initialize] = Initialize.default() @classmethod - def from_dict(cls, config_dict) -> "SnowflakeDynamicTableConfig": + def from_dict(cls, config_dict: Dict[str, Any]) -> Self: kwargs_dict = { "name": cls._render_part(ComponentName.Identifier, config_dict.get("name")), "schema_name": cls._render_part(ComponentName.Schema, config_dict.get("schema_name")), @@ -69,12 +75,12 @@ def from_dict(cls, config_dict) -> "SnowflakeDynamicTableConfig": "query": config_dict.get("query"), "target_lag": config_dict.get("target_lag"), "snowflake_warehouse": config_dict.get("snowflake_warehouse"), + "catalog": SnowflakeCatalogConfig.from_dict(config_dict["catalog"]), "refresh_mode": config_dict.get("refresh_mode"), "initialize": config_dict.get("initialize"), } - dynamic_table: "SnowflakeDynamicTableConfig" = super().from_dict(kwargs_dict) - return dynamic_table + return super().from_dict(kwargs_dict) @classmethod def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: @@ -85,18 +91,19 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any "query": relation_config.compiled_code, "target_lag": relation_config.config.extra.get("target_lag"), "snowflake_warehouse": relation_config.config.extra.get("snowflake_warehouse"), + "catalog": SnowflakeCatalogConfig.parse_relation_config(relation_config), } if refresh_mode := relation_config.config.extra.get("refresh_mode"): - config_dict.update(refresh_mode=refresh_mode.upper()) + config_dict["refresh_mode"] = refresh_mode.upper() if initialize := relation_config.config.extra.get("initialize"): - config_dict.update(initialize=initialize.upper()) + config_dict["initialize"] = initialize.upper() return config_dict @classmethod - def parse_relation_results(cls, relation_results: RelationResults) -> Dict: + def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str, Any]: dynamic_table: "agate.Row" = relation_results["dynamic_table"].rows[0] config_dict = { @@ -106,6 +113,7 @@ def parse_relation_results(cls, relation_results: RelationResults) -> Dict: "query": dynamic_table.get("text"), "target_lag": dynamic_table.get("target_lag"), "snowflake_warehouse": dynamic_table.get("warehouse"), + "catalog": SnowflakeCatalogConfig.parse_relation_results(relation_results), "refresh_mode": dynamic_table.get("refresh_mode"), # we don't get initialize since that's a one-time scheduler attribute, not a DT attribute } @@ -145,6 +153,7 @@ class SnowflakeDynamicTableConfigChangeset: target_lag: Optional[SnowflakeDynamicTableTargetLagConfigChange] = None snowflake_warehouse: Optional[SnowflakeDynamicTableWarehouseConfigChange] = None refresh_mode: Optional[SnowflakeDynamicTableRefreshModeConfigChange] = None + catalog: Optional[SnowflakeCatalogConfigChange] = None @property def requires_full_refresh(self) -> bool: @@ -157,9 +166,10 @@ def requires_full_refresh(self) -> bool: else False ), self.refresh_mode.requires_full_refresh if self.refresh_mode else False, + self.catalog.requires_full_refresh if self.catalog else False, ] ) @property def has_changes(self) -> bool: - return any([self.target_lag, self.snowflake_warehouse, self.refresh_mode]) + return any([self.target_lag, self.snowflake_warehouse, self.refresh_mode, self.catalog]) diff --git a/dbt/adapters/snowflake/relation_configs/formats.py b/dbt/adapters/snowflake/relation_configs/formats.py index 460241d9d..b6bb0bdda 100644 --- a/dbt/adapters/snowflake/relation_configs/formats.py +++ b/dbt/adapters/snowflake/relation_configs/formats.py @@ -1,4 +1,5 @@ from dbt_common.dataclass_schema import StrEnum # doesn't exist in standard library until py3.11 +from typing_extensions import Self class TableFormat(StrEnum): @@ -10,5 +11,9 @@ class TableFormat(StrEnum): DEFAULT = "default" ICEBERG = "iceberg" + @classmethod + def default(cls) -> Self: + return cls("default") + def __str__(self): return self.value diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql index 253788779..0bd190dcc 100644 --- a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql +++ b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql @@ -1,16 +1,83 @@ {% macro snowflake__get_create_dynamic_table_as_sql(relation, sql) -%} +{#- +-- Produce DDL that creates a dynamic table +-- +-- Args: +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Globals: +-- - config: NodeConfig - contains the attribution required to produce a SnowflakeDynamicTableConfig +-- Returns: +-- A valid DDL statement which will result in a new dynamic table. +-#} {%- set dynamic_table = relation.from_config(config.model) -%} + {%- if dynamic_table.catalog.table_format == 'iceberg' -%} + {{ _get_create_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) }} + {%- else -%} + {{ _get_create_dynamic_standard_table_as_sql(dynamic_table, relation, sql) }} + {%- endif -%} + +{%- endmacro %} + + +{% macro _get_create_dynamic_standard_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that creates a standard dynamic table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#syntax +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic standard table. +-#} + create dynamic table {{ relation }} target_lag = '{{ dynamic_table.target_lag }}' warehouse = {{ dynamic_table.snowflake_warehouse }} - {% if dynamic_table.refresh_mode %} - refresh_mode = {{ dynamic_table.refresh_mode }} - {% endif %} - {% if dynamic_table.initialize %} - initialize = {{ dynamic_table.initialize }} - {% endif %} + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} + as ( + {{ sql }} + ) + +{%- endmacro %} + + +{% macro _get_create_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that creates a dynamic iceberg table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic iceberg table. +-#} + + create dynamic iceberg table {{ relation }} + target_lag = '{{ dynamic_table.target_lag }}' + warehouse = {{ dynamic_table.snowflake_warehouse }} + {{ optional('external_volume', dynamic_table.catalog.external_volume) }} + {{ optional('catalog', dynamic_table.catalog.name) }} + base_location = {{ dynamic_table.catalog.base_location }} + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} as ( {{ sql }} ) diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql b/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql index cc79328fe..b5c49ad37 100644 --- a/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql +++ b/dbt/include/snowflake/macros/relations/dynamic_table/describe.sql @@ -1,4 +1,14 @@ {% macro snowflake__describe_dynamic_table(relation) %} +{#- +-- Get all relevant metadata about a dynamic table +-- +-- Args: +-- - relation: SnowflakeRelation - the relation to describe +-- Returns: +-- A dictionary with one or two entries depending on whether iceberg is enabled: +-- - dynamic_table: the metadata associated with a standard dynamic table +-- - catalog: the metadata associated with the iceberg catalog +-#} {%- set _dynamic_table_sql -%} show dynamic tables like '{{ relation.identifier }}' @@ -14,7 +24,32 @@ "refresh_mode" from table(result_scan(last_query_id())) {%- endset %} - {% set _dynamic_table = run_query(_dynamic_table_sql) %} + {% set results = {'dynamic_table': run_query(_dynamic_table_sql)} %} - {% do return({'dynamic_table': _dynamic_table}) %} + {% if adapter.behavior.enable_iceberg_materializations.no_warn %} + {% set _ = results.update({'catalog': run_query(_get_describe_iceberg_catalog_sql(relation))}) %} + {% endif %} + + {% do return(results) %} +{% endmacro %} + + +{% macro _get_describe_iceberg_catalog_sql(relation) %} +{#- +-- Produce DQL that returns all relevant metadata about an iceberg catalog +-- +-- Args: +-- - relation: SnowflakeRelation - the relation to describe +-- Returns: +-- A valid DQL statement that will return metadata associated with an iceberg catalog +-#} + show iceberg tables + like '{{ relation.identifier }}' + in schema {{ relation.database }}.{{ relation.schema }} + ; + select + "catalog_name", + "external_volume_name", + "base_location" + from table(result_scan(last_query_id())) {% endmacro %} diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql b/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql index dbe27d66e..f9ba1275a 100644 --- a/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql +++ b/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql @@ -1,16 +1,82 @@ {% macro snowflake__get_replace_dynamic_table_sql(relation, sql) -%} +{#- +-- Produce DDL that replaces a dynamic table with a new dynamic table +-- +-- Args: +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Globals: +-- - config: NodeConfig - contains the attribution required to produce a SnowflakeDynamicTableConfig +-- Returns: +-- A valid DDL statement which will result in a new dynamic table. +-#} {%- set dynamic_table = relation.from_config(config.model) -%} + {%- if dynamic_table.catalog.table_format == 'iceberg' -%} + {{ _get_replace_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) }} + {%- else -%} + {{ _get_replace_dynamic_standard_table_as_sql(dynamic_table, relation, sql) }} + {%- endif -%} + +{%- endmacro %} + +{% macro _get_replace_dynamic_standard_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that replaces a standard dynamic table with a new standard dynamic table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#syntax +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic standard table. +-#} + create or replace dynamic table {{ relation }} target_lag = '{{ dynamic_table.target_lag }}' warehouse = {{ dynamic_table.snowflake_warehouse }} - {% if dynamic_table.refresh_mode %} - refresh_mode = {{ dynamic_table.refresh_mode }} - {% endif %} - {% if dynamic_table.initialize %} - initialize = {{ dynamic_table.initialize }} - {% endif %} + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} + as ( + {{ sql }} + ) + +{%- endmacro %} + + +{% macro _get_replace_dynamic_iceberg_table_as_sql(dynamic_table, relation, sql) -%} +{#- +-- Produce DDL that replaces a dynamic iceberg table with a new dynamic iceberg table +-- +-- This follows the syntax outlined here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#create-dynamic-iceberg-table +-- +-- Args: +-- - dynamic_table: SnowflakeDynamicTableConfig - contains all of the configuration for the dynamic table +-- - relation: Union[SnowflakeRelation, str] +-- - SnowflakeRelation - required for relation.render() +-- - str - is already the rendered relation name +-- - sql: str - the code defining the model +-- Returns: +-- A valid DDL statement which will result in a new dynamic iceberg table. +-#} + + create or replace dynamic iceberg table {{ relation }} + target_lag = '{{ dynamic_table.target_lag }}' + warehouse = {{ dynamic_table.snowflake_warehouse }} + {{ optional('external_volume', dynamic_table.catalog.external_volume) }} + {{ optional('catalog', dynamic_table.catalog.name) }} + base_location = {{ dynamic_table.catalog.base_location }} + {{ optional('refresh_mode', dynamic_table.refresh_mode) }} + {{ optional('initialize', dynamic_table.initialize) }} as ( {{ sql }} ) diff --git a/dbt/include/snowflake/macros/utils/optional.sql b/dbt/include/snowflake/macros/utils/optional.sql new file mode 100644 index 000000000..0758ca59f --- /dev/null +++ b/dbt/include/snowflake/macros/utils/optional.sql @@ -0,0 +1,14 @@ +{% macro optional(name, value, quote_char = '') %} +{#- +-- Insert optional DDL parameters only when their value is provided; makes DDL statements more readable +-- +-- Args: +-- - name: the name of the DDL option +-- - value: the value of the DDL option, may be None +-- - quote_char: the quote character to use (e.g. string), leave blank if unnecessary (e.g. integer or bool) +-- Returns: +-- If the value is not None (e.g. provided by the user), return the option setting DDL +-- If the value is None, return an empty string +-#} +{% if value is not none %}{{ name }} = {{ quote_char }}{{ value }}{{ quote_char }}{% endif %} +{% endmacro %} diff --git a/tests/functional/relation_tests/dynamic_table_tests/models.py b/tests/functional/relation_tests/dynamic_table_tests/models.py index 5e46bed53..4dcd6cf48 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/models.py +++ b/tests/functional/relation_tests/dynamic_table_tests/models.py @@ -10,7 +10,7 @@ {{ config( materialized='dynamic_table', snowflake_warehouse='DBT_TESTING', - target_lag='2 minutes', + target_lag='2 minutes', refresh_mode='INCREMENTAL', ) }} select * from {{ ref('my_seed') }} @@ -28,11 +28,25 @@ """ +DYNAMIC_ICEBERG_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='INCREMENTAL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" + + DYNAMIC_TABLE_ALTER = """ {{ config( materialized='dynamic_table', snowflake_warehouse='DBT_TESTING', - target_lag='5 minutes', + target_lag='5 minutes', refresh_mode='INCREMENTAL', ) }} select * from {{ ref('my_seed') }} @@ -43,8 +57,36 @@ {{ config( materialized='dynamic_table', snowflake_warehouse='DBT_TESTING', - target_lag='2 minutes', + target_lag='2 minutes', + refresh_mode='FULL', +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_ICEBERG_TABLE_ALTER = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='5 minutes', + refresh_mode='INCREMENTAL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", +) }} +select * from {{ ref('my_seed') }} +""" + + +DYNAMIC_ICEBERG_TABLE_REPLACE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', refresh_mode='FULL', + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", ) }} select * from {{ ref('my_seed') }} """ diff --git a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py index 2406e1c14..79a2241ca 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py +++ b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py @@ -7,6 +7,7 @@ class TestBasic: + iceberg: bool = False @pytest.fixture(scope="class", autouse=True) def seeds(self): @@ -14,10 +15,17 @@ def seeds(self): @pytest.fixture(scope="class", autouse=True) def models(self): - yield { + my_models = { "my_dynamic_table.sql": models.DYNAMIC_TABLE, "my_dynamic_table_downstream.sql": models.DYNAMIC_TABLE_DOWNSTREAM, } + if self.iceberg: + my_models.update( + { + "my_dynamic_iceberg_table.sql": models.DYNAMIC_ICEBERG_TABLE, + } + ) + yield my_models @pytest.fixture(scope="class", autouse=True) def setup(self, project): @@ -28,3 +36,13 @@ def test_dynamic_table_full_refresh(self, project): run_dbt(["run", "--full-refresh"]) assert query_relation_type(project, "my_dynamic_table") == "dynamic_table" assert query_relation_type(project, "my_dynamic_table_downstream") == "dynamic_table" + if self.iceberg: + assert query_relation_type(project, "my_dynamic_iceberg_table") == "dynamic_table" + + +class TestBasicIcebergOn(TestBasic): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} diff --git a/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py b/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py index 3c4f65a87..f389344e0 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py +++ b/tests/functional/relation_tests/dynamic_table_tests/test_configuration_changes.py @@ -7,6 +7,7 @@ class Changes: + iceberg: bool = False @pytest.fixture(scope="class", autouse=True) def seeds(self): @@ -14,10 +15,18 @@ def seeds(self): @pytest.fixture(scope="class", autouse=True) def models(self): - yield { + my_models = { "dynamic_table_alter.sql": models.DYNAMIC_TABLE, "dynamic_table_replace.sql": models.DYNAMIC_TABLE, } + if self.iceberg: + my_models.update( + { + "dynamic_table_iceberg_alter.sql": models.DYNAMIC_ICEBERG_TABLE, + "dynamic_table_iceberg_replace.sql": models.DYNAMIC_ICEBERG_TABLE, + } + ) + yield my_models @pytest.fixture(scope="function", autouse=True) def setup_class(self, project): @@ -33,14 +42,23 @@ def setup_method(self, project, setup_class): update_model(project, "dynamic_table_alter", models.DYNAMIC_TABLE_ALTER) update_model(project, "dynamic_table_replace", models.DYNAMIC_TABLE_REPLACE) + if self.iceberg: + update_model( + project, "dynamic_table_iceberg_alter", models.DYNAMIC_ICEBERG_TABLE_ALTER + ) + update_model( + project, "dynamic_table_iceberg_replace", models.DYNAMIC_ICEBERG_TABLE_REPLACE + ) yield update_model(project, "dynamic_table_alter", models.DYNAMIC_TABLE) update_model(project, "dynamic_table_replace", models.DYNAMIC_TABLE) + if self.iceberg: + update_model(project, "dynamic_table_iceberg_alter", models.DYNAMIC_ICEBERG_TABLE) + update_model(project, "dynamic_table_iceberg_replace", models.DYNAMIC_ICEBERG_TABLE) - @staticmethod - def assert_changes_are_applied(project): + def assert_changes_are_applied(self, project): altered = describe_dynamic_table(project, "dynamic_table_alter") assert altered.snowflake_warehouse == "DBT_TESTING" assert altered.target_lag == "5 minutes" # this updated @@ -51,8 +69,18 @@ def assert_changes_are_applied(project): assert replaced.target_lag == "2 minutes" assert replaced.refresh_mode == "FULL" # this updated - @staticmethod - def assert_changes_are_not_applied(project): + if self.iceberg: + altered_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_alter") + assert altered_iceberg.snowflake_warehouse == "DBT_TESTING" + assert altered_iceberg.target_lag == "5 minutes" # this updated + assert altered_iceberg.refresh_mode == "INCREMENTAL" + + replaced_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_replace") + assert replaced_iceberg.snowflake_warehouse == "DBT_TESTING" + assert replaced_iceberg.target_lag == "2 minutes" + assert replaced_iceberg.refresh_mode == "FULL" # this updated + + def assert_changes_are_not_applied(self, project): altered = describe_dynamic_table(project, "dynamic_table_alter") assert altered.snowflake_warehouse == "DBT_TESTING" assert altered.target_lag == "2 minutes" # this would have updated, but didn't @@ -63,6 +91,19 @@ def assert_changes_are_not_applied(project): assert replaced.target_lag == "2 minutes" assert replaced.refresh_mode == "INCREMENTAL" # this would have updated, but didn't + if self.iceberg: + altered_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_alter") + assert altered_iceberg.snowflake_warehouse == "DBT_TESTING" + assert altered_iceberg.target_lag == "2 minutes" # this would have updated, but didn't + assert altered_iceberg.refresh_mode == "INCREMENTAL" + + replaced_iceberg = describe_dynamic_table(project, "dynamic_table_iceberg_replace") + assert replaced_iceberg.snowflake_warehouse == "DBT_TESTING" + assert replaced_iceberg.target_lag == "2 minutes" + assert ( + replaced_iceberg.refresh_mode == "INCREMENTAL" + ) # this would have updated, but didn't + def test_full_refresh_is_always_successful(self, project): # this always passes and always changes the configuration, regardless of on_configuration_change # and regardless of whether the changes require a replace versus an alter @@ -81,6 +122,17 @@ def test_changes_are_applied(self, project): self.assert_changes_are_applied(project) +class TestChangesApplyIcebergOn(TestChangesApply): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": {"on_configuration_change": "apply"}, + "flags": {"enable_iceberg_materializations": True}, + } + + class TestChangesContinue(Changes): @pytest.fixture(scope="class") def project_config_update(self): @@ -92,6 +144,17 @@ def test_changes_are_not_applied(self, project): self.assert_changes_are_not_applied(project) +class TestChangesContinueIcebergOn(TestChangesContinue): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": {"on_configuration_change": "continue"}, + "flags": {"enable_iceberg_materializations": True}, + } + + class TestChangesFail(Changes): @pytest.fixture(scope="class") def project_config_update(self): @@ -101,3 +164,14 @@ def test_changes_are_not_applied(self, project): # this fails and does not change the configuration run_dbt(["run"], expect_pass=False) self.assert_changes_are_not_applied(project) + + +class TestChangesFailIcebergOn(TestChangesFail): + iceberg = True + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": {"on_configuration_change": "fail"}, + "flags": {"enable_iceberg_materializations": True}, + } diff --git a/tests/functional/relation_tests/models.py b/tests/functional/relation_tests/models.py index 63dfff045..7b0050d11 100644 --- a/tests/functional/relation_tests/models.py +++ b/tests/functional/relation_tests/models.py @@ -55,7 +55,7 @@ select * from {{ ref('my_seed') }} """ -ICEBERG_INCREMENTAL_TABLE = """ +INCREMENTAL_ICEBERG_TABLE = """ {{ config( materialized='incremental', table_format='iceberg', @@ -65,3 +65,13 @@ ) }} select * from {{ ref('my_seed') }} """ + + +INCREMENTAL_TABLE = """ +{{ config( + materialized='incremental', + incremental_strategy='append', + unique_key="id", +) }} +select * from {{ ref('my_seed') }} +""" diff --git a/tests/functional/relation_tests/test_relation_type_change.py b/tests/functional/relation_tests/test_relation_type_change.py index c2886ad04..1024a92ca 100644 --- a/tests/functional/relation_tests/test_relation_type_change.py +++ b/tests/functional/relation_tests/test_relation_type_change.py @@ -13,16 +13,25 @@ class Model: model: str relation_type: str - table_format: Optional[str] = None - incremental: Optional[bool] = None + table_format: Optional[str] = "default" + is_incremental: Optional[bool] = False @property def name(self) -> str: - name = f"{self.relation_type}" - if self.table_format: - name += f"_{self.table_format}" + if self.is_incremental: + name = f"{self.relation_type}_{self.table_format}_incremental" + else: + name = f"{self.relation_type}_{self.table_format}" return name + @property + def is_iceberg(self) -> bool: + return self.table_format == "iceberg" + + @property + def is_standard_table(self) -> bool: + return self.relation_type == "table" and not self.is_incremental + @dataclass class Scenario: @@ -37,26 +46,47 @@ def name(self) -> str: def error_message(self) -> str: return f"Failed when migrating from: {self.initial.name} to: {self.final.name}" + @property + def uses_iceberg(self) -> bool: + return any([self.initial.is_iceberg, self.final.is_iceberg]) + relations = [ Model(models.VIEW, "view"), Model(models.TABLE, "table", "default"), - # to be activated upon merge of dynamic table support PR - # Model(models.DYNAMIC_TABLE, "dynamic_table", "default"), - # Model(models.DYNAMIC_ICEBERG_TABLE, "dynamic_table", "iceberg"), + Model(models.INCREMENTAL_TABLE, "table", "default", is_incremental=True), + Model(models.DYNAMIC_TABLE, "dynamic_table", "default"), Model(models.ICEBERG_TABLE, "table", "iceberg"), - Model(models.ICEBERG_INCREMENTAL_TABLE, "table", "iceberg", incremental=True), + Model(models.INCREMENTAL_ICEBERG_TABLE, "table", "iceberg", is_incremental=True), + Model(models.DYNAMIC_ICEBERG_TABLE, "dynamic_table", "iceberg"), ] scenarios = [Scenario(*scenario) for scenario in product(relations, relations)] +def requires_full_refresh(scenario) -> bool: + return any( + [ + # we can only swap incremental to table and back if both are iceberg + scenario.initial.is_incremental + and scenario.final.is_standard_table + and scenario.initial.table_format != scenario.final.table_format, + scenario.initial.is_standard_table + and scenario.final.is_incremental + and scenario.initial.table_format != scenario.final.table_format, + # we can't swap from an incremental to a dynamic table because the materialization does not handle this case + scenario.initial.relation_type == "dynamic_table" and scenario.final.is_incremental, + ] + ) + + class TestRelationTypeChange: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": False}} @staticmethod def include(scenario) -> bool: - return ( - scenario.initial.table_format != "iceberg" and scenario.final.table_format != "iceberg" - ) + return not scenario.uses_iceberg and not requires_full_refresh(scenario) @pytest.fixture(scope="class", autouse=True) def seeds(self): @@ -77,7 +107,11 @@ def setup(self, project): for scenario in scenarios: if self.include(scenario): update_model(project, scenario.name, scenario.final.model) - run_dbt(["run"]) + # allow for dbt to fail so that we can see which scenarios pass and which scenarios fail + try: + run_dbt(["run"], expect_pass=False) + except Exception: + pass @pytest.mark.parametrize("scenario", scenarios, ids=[scenario.name for scenario in scenarios]) def test_replace(self, project, scenario): @@ -91,9 +125,17 @@ def test_replace(self, project, scenario): pytest.skip() -""" -Upon adding the logic needed for seamless transitions to and from incremental models without data loss, we can coalesce these test cases. -""" +class TestRelationTypeChangeFullRefreshRequired(TestRelationTypeChange): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": {"enable_iceberg_materializations": False}, + "models": {"full_refresh": True}, + } + + @staticmethod + def include(scenario) -> bool: + return not scenario.uses_iceberg and requires_full_refresh(scenario) class TestRelationTypeChangeIcebergOn(TestRelationTypeChange): @@ -103,21 +145,17 @@ def project_config_update(self): @staticmethod def include(scenario) -> bool: - return any( - ( - # scenario 1: Everything that doesn't include incremental relations on Iceberg - ( - ( - scenario.initial.table_format == "iceberg" - or scenario.final.table_format == "iceberg" - ) - and not scenario.initial.incremental - and not scenario.final.incremental - ), - # scenario 2: Iceberg Incremental swaps allowed - ( - scenario.initial.table_format == "iceberg" - and scenario.final.table_format == "iceberg" - ), - ) - ) + return scenario.uses_iceberg and not requires_full_refresh(scenario) + + +class TestRelationTypeChangeIcebergOnFullRefreshRequired(TestRelationTypeChange): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": {"enable_iceberg_materializations": True}, + "models": {"full_refresh": True}, + } + + @staticmethod + def include(scenario) -> bool: + return scenario.uses_iceberg and requires_full_refresh(scenario) From 5b595fb0a061aad69f15fff93d6c5c458c15ec7f Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:58:06 -0400 Subject: [PATCH 13/41] [Bug] Fix issue where `dbt-snowflake` attempts to drop database roles during grants sync (#1188) * publish failed reproduction case * remove database roles when standardizing the grants dict so that dbt-snowflake does not attempt to revoke them --- .../unreleased/Fixes-20240920-193613.yaml | 6 ++ dbt/adapters/snowflake/impl.py | 2 +- .../auth_tests/test_database_role.py | 68 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20240920-193613.yaml create mode 100644 tests/functional/auth_tests/test_database_role.py diff --git a/.changes/unreleased/Fixes-20240920-193613.yaml b/.changes/unreleased/Fixes-20240920-193613.yaml new file mode 100644 index 000000000..f85f6fc56 --- /dev/null +++ b/.changes/unreleased/Fixes-20240920-193613.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix issue where dbt-snowflake attempts to drop database roles during grants sync +time: 2024-09-20T19:36:13.671173-04:00 +custom: + Author: mikealfare + Issue: "1151" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 5b5881eed..6320893e1 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -320,7 +320,7 @@ def standardize_grants_dict(self, grants_table: "agate.Table") -> dict: grantee = row["grantee_name"] granted_to = row["granted_to"] privilege = row["privilege"] - if privilege != "OWNERSHIP" and granted_to != "SHARE": + if privilege != "OWNERSHIP" and granted_to not in ["SHARE", "DATABASE_ROLE"]: if privilege in grants_dict.keys(): grants_dict[privilege].append(grantee) else: diff --git a/tests/functional/auth_tests/test_database_role.py b/tests/functional/auth_tests/test_database_role.py new file mode 100644 index 000000000..f36555e34 --- /dev/null +++ b/tests/functional/auth_tests/test_database_role.py @@ -0,0 +1,68 @@ +import os + +import pytest + +from dbt.tests.util import run_dbt + + +SEED = """ +id +1 +""".strip() + + +MODEL = """ +{{ config( + materialized='incremental', +) }} +select * from {{ ref('my_seed') }} +""" + + +class TestDatabaseRole: + """ + This test addresses https://github.com/dbt-labs/dbt-snowflake/issues/1151 + + While dbt-snowflake does not manage database roles (it only manages account roles, + it still needs to account for them so that it doesn't try to revoke them. + """ + + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": SEED} + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": MODEL} + + @pytest.fixture(scope="class") + def project_config_update(self): + # grant to the test role even though this role already has these permissions + # this triggers syncing grants since `apply_grants` first looks for a grants config + return {"models": {"+grants": {"select": [os.getenv("SNOWFLAKE_TEST_ROLE")]}}} + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + """ + Create a database role with access to the model we're about to create. + The existence of this database role triggered the bug as dbt-snowflake attempts + to revoke it if the user also provides a grants config. + """ + role = "BLOCKING_DB_ROLE" + project.run_sql(f"CREATE DATABASE ROLE {role}") + sql = f""" + GRANT + ALL PRIVILEGES ON FUTURE TABLES + IN DATABASE {project.database} + TO DATABASE ROLE {role} + """ + project.run_sql(sql) + yield + project.run_sql(f"DROP DATABASE ROLE {role}") + + def test_database_role(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + # run a second time to trigger revoke on an incremental update + # this originally failed, demonstrating the bug + run_dbt(["run"]) From f60c476596ea45f609badc3e9bba06c9397ee5d0 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:02:42 -0700 Subject: [PATCH 14/41] Adap 321/support all on schema change options (#1195) * Add iceberg ddl generation --------- Co-authored-by: VersusFacit --- .changes/unreleased/Features-20240930-112041.yaml | 6 ++++++ dbt/include/snowflake/macros/adapters.sql | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Features-20240930-112041.yaml diff --git a/.changes/unreleased/Features-20240930-112041.yaml b/.changes/unreleased/Features-20240930-112041.yaml new file mode 100644 index 000000000..1395a8bf7 --- /dev/null +++ b/.changes/unreleased/Features-20240930-112041.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for all on_schema_change incremental model strategies. +time: 2024-09-30T11:20:41.99589-07:00 +custom: + Author: versusfacit + Issue: "321" diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index aa8895819..b60cea0b0 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -195,7 +195,7 @@ {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} {% call statement('alter_column_type') %} - alter table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; + alter {{ relation.get_ddl_prefix_for_alter() }} table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; {% endcall %} {% endmacro %} @@ -216,7 +216,7 @@ {% else -%} {% set relation_type = relation.type %} {% endif %} - alter {{ relation_type }} {{ relation.render() }} alter + alter {{ relation.get_ddl_prefix_for_alter() }} {{ relation_type }} {{ relation.render() }} alter {% for column_name in existing_columns if (column_name in existing_columns) or (column_name|lower in existing_columns) %} {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last else ';' }} {% endfor %} @@ -275,7 +275,7 @@ {% if add_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation.render() }} add column + alter {{ relation.get_ddl_prefix_for_alter() }} {{ relation_type }} {{ relation.render() }} add column {% for column in add_columns %} {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %} @@ -288,7 +288,7 @@ {% if remove_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation.render() }} drop column + alter {{ relation.get_ddl_prefix_for_alter() }} {{ relation_type }} {{ relation.render() }} drop column {% for column in remove_columns %} {{ column.name }}{{ ',' if not loop.last }} {% endfor %} From eea98443b21f6f3bdc3d297b727a0a9424d54d7b Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:58:01 -0700 Subject: [PATCH 15/41] Add test for 321/support all on schema change options (#1196) * Add iceberg ddl generation. * Add changelog. * add test. --------- Co-authored-by: VersusFacit --- .../iceberg/test_incremental_models.py | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/functional/iceberg/test_incremental_models.py b/tests/functional/iceberg/test_incremental_models.py index f8f1d6b89..35ccdcd89 100644 --- a/tests/functional/iceberg/test_incremental_models.py +++ b/tests/functional/iceberg/test_incremental_models.py @@ -3,7 +3,7 @@ from pathlib import Path -from dbt.tests.util import run_dbt, run_dbt_and_capture +from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file _SEED_INCREMENTAL_STRATEGIES = """ @@ -32,6 +32,7 @@ incremental_strategy='{strategy}', unique_key="world_id", external_volume = "s3_iceberg_snow", + on_schema_change = "sync_all_columns" ) }}}} select * from {{{{ ref('upstream_table') }}}} @@ -123,3 +124,46 @@ def test_incremental_strategies_with_update(self, project, setup_class): self.__check_correct_operations(self.append, rows_affected=2) self.__check_correct_operations("merge", rows_affected=1) self.__check_correct_operations("delete_insert", rows_affected=1) + + +class TestIcebergIncrementalOnSchemaChangeMutatesRelations: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEED_INCREMENTAL_STRATEGIES, + } + + @pytest.fixture(scope="function", autouse=True) + def setup_class(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield + + @pytest.fixture(scope="class") + def models(self): + return { + "upstream_table.sql": _MODEL_BASIC_TABLE_MODEL, + "merge.sql": _MODEL_INCREMENTAL_ICEBERG_MERGE, + } + + def test_sync_and_append_semantics(self, project, setup_class): + model_file = project.project_root / Path("models") / Path("merge.sql") + sql = f"show columns in {project.database}.{project.test_schema}.merge;" + column_names = [column[2] for column in project.run_sql(sql, fetch="all")] + assert len(column_names) == 3 + + write_file(_MODEL_INCREMENTAL_ICEBERG_MERGE.replace("*", "*, 1 as new_column"), model_file) + run_dbt() + column_names = [column[2].lower() for column in project.run_sql(sql, fetch="all")] + assert len(column_names) == 4 + assert "new_column" in column_names + + write_file(_MODEL_INCREMENTAL_ICEBERG_MERGE, model_file) + run_dbt() + column_names = [column[2].lower() for column in project.run_sql(sql, fetch="all")] + assert len(column_names) == 3 + assert "new_column" not in column_names From 139f5c3e2559587cc06890f9448c4afdab49f275 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 1 Oct 2024 19:30:33 +0000 Subject: [PATCH 16/41] Bumping version to 1.9.0b1 and generate changelog --- .bumpversion.cfg | 2 +- .changes/1.9.0-b1.md | 61 +++++++++++++++++ .../Dependencies-20231219-125152.yaml | 0 .../Dependencies-20240412-155921.yaml | 0 .../Dependencies-20240429-124038.yaml | 0 .../Dependencies-20240429-124044.yaml | 0 .../Dependencies-20240624-122538.yaml | 0 .../Dependencies-20240718-120848.yaml | 0 .../Dependencies-20240718-120849.yaml | 0 .../Dependencies-20240718-120852.yaml | 0 .../Dependencies-20240718-120857.yaml | 0 .../Dependencies-20240719-120828.yaml | 0 .../Features-20240131-125318.yaml | 0 .../Features-20240430-185714.yaml | 0 .../Features-20240501-151901.yaml | 0 .../Features-20240604-154856.yaml | 0 .../Features-20240610-171026.yaml | 0 .../Features-20240709-194316.yaml | 0 .../Features-20240710-172345.yaml | 0 .../Features-20240911-001806.yaml | 0 .../Features-20240913-215416.yaml | 0 .../Features-20240917-100505.yaml | 0 .../Features-20240923-203204.yaml | 0 .../Features-20240930-112041.yaml | 0 .../Fixes-20240516-174337.yaml | 0 .../Fixes-20240516-224134.yaml | 0 .../Fixes-20240522-160538.yaml | 0 .../Fixes-20240605-125611.yaml | 0 .../Fixes-20240607-102708.yaml | 0 .../Fixes-20240628-190140.yaml | 0 .../Fixes-20240705-165932.yaml | 0 .../Fixes-20240920-193613.yaml | 0 .../Under the Hood-20240327-001304.yaml | 0 .../Under the Hood-20240425-144556.yaml | 0 .../Under the Hood-20240517-143743.yaml | 0 .../Under the Hood-20240614-170858.yaml | 0 .../Under the Hood-20240716-174655.yaml | 0 .../Under the Hood-20240719-125618.yaml | 0 .../Under the Hood-20240722-143114.yaml | 0 .../Under the Hood-20240806-215935.yaml | 0 .../Under the Hood-20240917-181147.yaml | 0 CHANGELOG.md | 65 ++++++++++++++++++- dbt/adapters/snowflake/__version__.py | 2 +- 43 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 .changes/1.9.0-b1.md rename .changes/{unreleased => 1.9.0}/Dependencies-20231219-125152.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240412-155921.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240429-124038.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240429-124044.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240624-122538.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-120848.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-120849.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-120852.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-120857.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240719-120828.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240131-125318.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240430-185714.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240501-151901.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240604-154856.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240610-171026.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240709-194316.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240710-172345.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240911-001806.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240913-215416.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240917-100505.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240923-203204.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240930-112041.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240516-174337.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240516-224134.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240522-160538.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240605-125611.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240607-102708.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240628-190140.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240705-165932.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240920-193613.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240327-001304.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240425-144556.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240517-143743.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240614-170858.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240716-174655.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240719-125618.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240722-143114.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240806-215935.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240917-181147.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0fa84469a..7ff98322d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0a1 +current_version = 1.9.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md new file mode 100644 index 000000000..15a01afe7 --- /dev/null +++ b/.changes/1.9.0-b1.md @@ -0,0 +1,61 @@ +## dbt-snowflake 1.9.0-b1 - October 01, 2024 + +### Features + +- Support refresh_mode and initialize parameters for dynamic tables ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) +- Add tests for cross-database `cast` macro ([#1009](https://github.com/dbt-labs/dbt-snowflake/issues/1009)) +- Cross-database `date` macro ([#1013](https://github.com/dbt-labs/dbt-snowflake/issues/1013)) +- Replace underscores with hyphens in account IDs to prevent SSL issues ([#1068](https://github.com/dbt-labs/dbt-snowflake/issues/1068)) +- Support JWT Authentication ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) +- Improve run times for large projects by reusing connections by default ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) +- Improve run times when using key pair auth by caching the private key ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) +- Add support for Iceberg table materializations. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) +- Microbatch incremental strategy ([#1182](https://github.com/dbt-labs/dbt-snowflake/issues/1182)) +- Add support for Iceberg table format in Dynamic Tables ([#1183](https://github.com/dbt-labs/dbt-snowflake/issues/1183)) +- Add Iceberg format Incremental Models ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) +- Add support for all on_schema_change incremental model strategies. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) + +### Fixes + +- Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability ([#1048](https://github.com/dbt-labs/dbt-snowflake/issues/1048)) +- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) +- Rename targets for tables and views use fully qualified names ([#1031](https://github.com/dbt-labs/dbt-snowflake/issues/1031)) +- Surface SSO token expiration in logs ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) +- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) +- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) +- Use show ... starts with instead of show ... like in _show_object_metadata ([#1102](https://github.com/dbt-labs/dbt-snowflake/issues/1102)) +- Fix issue where dbt-snowflake attempts to drop database roles during grants sync ([#1151](https://github.com/dbt-labs/dbt-snowflake/issues/1151)) + +### Under the Hood + +- Lazy load agate ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) +- Speedup catalog string comparison by using ilike before equals ([#1035](https://github.com/dbt-labs/dbt-snowflake/issues/1035)) +- Improve memory efficiency of the process_results() override. ([#1053](https://github.com/dbt-labs/dbt-snowflake/issues/1053)) +- Automate all manual integration tests for Dynamic Tables ([#1084](https://github.com/dbt-labs/dbt-snowflake/issues/1084)) +- Add support for experimental record/replay testing. ([#1106](https://github.com/dbt-labs/dbt-snowflake/issues/1106)) +- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-snowflake/issues/1136)) +- Add support for Python 3.12 ([#903](https://github.com/dbt-labs/dbt-snowflake/issues/903)) +- Isolating distribution testing ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) +- Change behavior flag semantics to log iceberg flag warnings.. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) + +### Dependencies + +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#869](https://github.com/dbt-labs/dbt-snowflake/pull/869)) +- Bump actions/upload-artifact from 3 to 4 ([#971](https://github.com/dbt-labs/dbt-snowflake/pull/971)) +- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1006](https://github.com/dbt-labs/dbt-snowflake/pull/1006)) +- Bump actions/download-artifact from 3 to 4 ([#1007](https://github.com/dbt-labs/dbt-snowflake/pull/1007)) +- Bump aurelien-baudet/workflow-dispatch from 2 to 4 ([#1093](https://github.com/dbt-labs/dbt-snowflake/pull/1093)) +- Update twine requirement from ~=4.0 to ~=5.1 ([#1120](https://github.com/dbt-labs/dbt-snowflake/pull/1120)) +- Bump pre-commit from 3.7.0 to 3.7.1 ([#1119](https://github.com/dbt-labs/dbt-snowflake/pull/1119)) +- Update wheel requirement from ~=0.42 to ~=0.43 ([#1121](https://github.com/dbt-labs/dbt-snowflake/pull/1121)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1122](https://github.com/dbt-labs/dbt-snowflake/pull/1122)) +- Update tox requirement from ~=4.11 to ~=4.16 ([#1135](https://github.com/dbt-labs/dbt-snowflake/pull/1135)) + +### Contributors +- [@HenkvanDyk,mikealfare](https://github.com/HenkvanDyk,mikealfare) ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) +- [@McKnight-42](https://github.com/McKnight-42) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851), [#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) +- [@amardatar](https://github.com/amardatar) ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) +- [@dwreeves](https://github.com/dwreeves) ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) +- [@leahwicz](https://github.com/leahwicz) ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) +- [@llam15](https://github.com/llam15) ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) +- [@mikealfare,](https://github.com/mikealfare,) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) diff --git a/.changes/unreleased/Dependencies-20231219-125152.yaml b/.changes/1.9.0/Dependencies-20231219-125152.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231219-125152.yaml rename to .changes/1.9.0/Dependencies-20231219-125152.yaml diff --git a/.changes/unreleased/Dependencies-20240412-155921.yaml b/.changes/1.9.0/Dependencies-20240412-155921.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240412-155921.yaml rename to .changes/1.9.0/Dependencies-20240412-155921.yaml diff --git a/.changes/unreleased/Dependencies-20240429-124038.yaml b/.changes/1.9.0/Dependencies-20240429-124038.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240429-124038.yaml rename to .changes/1.9.0/Dependencies-20240429-124038.yaml diff --git a/.changes/unreleased/Dependencies-20240429-124044.yaml b/.changes/1.9.0/Dependencies-20240429-124044.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240429-124044.yaml rename to .changes/1.9.0/Dependencies-20240429-124044.yaml diff --git a/.changes/unreleased/Dependencies-20240624-122538.yaml b/.changes/1.9.0/Dependencies-20240624-122538.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240624-122538.yaml rename to .changes/1.9.0/Dependencies-20240624-122538.yaml diff --git a/.changes/unreleased/Dependencies-20240718-120848.yaml b/.changes/1.9.0/Dependencies-20240718-120848.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-120848.yaml rename to .changes/1.9.0/Dependencies-20240718-120848.yaml diff --git a/.changes/unreleased/Dependencies-20240718-120849.yaml b/.changes/1.9.0/Dependencies-20240718-120849.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-120849.yaml rename to .changes/1.9.0/Dependencies-20240718-120849.yaml diff --git a/.changes/unreleased/Dependencies-20240718-120852.yaml b/.changes/1.9.0/Dependencies-20240718-120852.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-120852.yaml rename to .changes/1.9.0/Dependencies-20240718-120852.yaml diff --git a/.changes/unreleased/Dependencies-20240718-120857.yaml b/.changes/1.9.0/Dependencies-20240718-120857.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-120857.yaml rename to .changes/1.9.0/Dependencies-20240718-120857.yaml diff --git a/.changes/unreleased/Dependencies-20240719-120828.yaml b/.changes/1.9.0/Dependencies-20240719-120828.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240719-120828.yaml rename to .changes/1.9.0/Dependencies-20240719-120828.yaml diff --git a/.changes/unreleased/Features-20240131-125318.yaml b/.changes/1.9.0/Features-20240131-125318.yaml similarity index 100% rename from .changes/unreleased/Features-20240131-125318.yaml rename to .changes/1.9.0/Features-20240131-125318.yaml diff --git a/.changes/unreleased/Features-20240430-185714.yaml b/.changes/1.9.0/Features-20240430-185714.yaml similarity index 100% rename from .changes/unreleased/Features-20240430-185714.yaml rename to .changes/1.9.0/Features-20240430-185714.yaml diff --git a/.changes/unreleased/Features-20240501-151901.yaml b/.changes/1.9.0/Features-20240501-151901.yaml similarity index 100% rename from .changes/unreleased/Features-20240501-151901.yaml rename to .changes/1.9.0/Features-20240501-151901.yaml diff --git a/.changes/unreleased/Features-20240604-154856.yaml b/.changes/1.9.0/Features-20240604-154856.yaml similarity index 100% rename from .changes/unreleased/Features-20240604-154856.yaml rename to .changes/1.9.0/Features-20240604-154856.yaml diff --git a/.changes/unreleased/Features-20240610-171026.yaml b/.changes/1.9.0/Features-20240610-171026.yaml similarity index 100% rename from .changes/unreleased/Features-20240610-171026.yaml rename to .changes/1.9.0/Features-20240610-171026.yaml diff --git a/.changes/unreleased/Features-20240709-194316.yaml b/.changes/1.9.0/Features-20240709-194316.yaml similarity index 100% rename from .changes/unreleased/Features-20240709-194316.yaml rename to .changes/1.9.0/Features-20240709-194316.yaml diff --git a/.changes/unreleased/Features-20240710-172345.yaml b/.changes/1.9.0/Features-20240710-172345.yaml similarity index 100% rename from .changes/unreleased/Features-20240710-172345.yaml rename to .changes/1.9.0/Features-20240710-172345.yaml diff --git a/.changes/unreleased/Features-20240911-001806.yaml b/.changes/1.9.0/Features-20240911-001806.yaml similarity index 100% rename from .changes/unreleased/Features-20240911-001806.yaml rename to .changes/1.9.0/Features-20240911-001806.yaml diff --git a/.changes/unreleased/Features-20240913-215416.yaml b/.changes/1.9.0/Features-20240913-215416.yaml similarity index 100% rename from .changes/unreleased/Features-20240913-215416.yaml rename to .changes/1.9.0/Features-20240913-215416.yaml diff --git a/.changes/unreleased/Features-20240917-100505.yaml b/.changes/1.9.0/Features-20240917-100505.yaml similarity index 100% rename from .changes/unreleased/Features-20240917-100505.yaml rename to .changes/1.9.0/Features-20240917-100505.yaml diff --git a/.changes/unreleased/Features-20240923-203204.yaml b/.changes/1.9.0/Features-20240923-203204.yaml similarity index 100% rename from .changes/unreleased/Features-20240923-203204.yaml rename to .changes/1.9.0/Features-20240923-203204.yaml diff --git a/.changes/unreleased/Features-20240930-112041.yaml b/.changes/1.9.0/Features-20240930-112041.yaml similarity index 100% rename from .changes/unreleased/Features-20240930-112041.yaml rename to .changes/1.9.0/Features-20240930-112041.yaml diff --git a/.changes/unreleased/Fixes-20240516-174337.yaml b/.changes/1.9.0/Fixes-20240516-174337.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240516-174337.yaml rename to .changes/1.9.0/Fixes-20240516-174337.yaml diff --git a/.changes/unreleased/Fixes-20240516-224134.yaml b/.changes/1.9.0/Fixes-20240516-224134.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240516-224134.yaml rename to .changes/1.9.0/Fixes-20240516-224134.yaml diff --git a/.changes/unreleased/Fixes-20240522-160538.yaml b/.changes/1.9.0/Fixes-20240522-160538.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240522-160538.yaml rename to .changes/1.9.0/Fixes-20240522-160538.yaml diff --git a/.changes/unreleased/Fixes-20240605-125611.yaml b/.changes/1.9.0/Fixes-20240605-125611.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240605-125611.yaml rename to .changes/1.9.0/Fixes-20240605-125611.yaml diff --git a/.changes/unreleased/Fixes-20240607-102708.yaml b/.changes/1.9.0/Fixes-20240607-102708.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240607-102708.yaml rename to .changes/1.9.0/Fixes-20240607-102708.yaml diff --git a/.changes/unreleased/Fixes-20240628-190140.yaml b/.changes/1.9.0/Fixes-20240628-190140.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240628-190140.yaml rename to .changes/1.9.0/Fixes-20240628-190140.yaml diff --git a/.changes/unreleased/Fixes-20240705-165932.yaml b/.changes/1.9.0/Fixes-20240705-165932.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240705-165932.yaml rename to .changes/1.9.0/Fixes-20240705-165932.yaml diff --git a/.changes/unreleased/Fixes-20240920-193613.yaml b/.changes/1.9.0/Fixes-20240920-193613.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240920-193613.yaml rename to .changes/1.9.0/Fixes-20240920-193613.yaml diff --git a/.changes/unreleased/Under the Hood-20240327-001304.yaml b/.changes/1.9.0/Under the Hood-20240327-001304.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240327-001304.yaml rename to .changes/1.9.0/Under the Hood-20240327-001304.yaml diff --git a/.changes/unreleased/Under the Hood-20240425-144556.yaml b/.changes/1.9.0/Under the Hood-20240425-144556.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240425-144556.yaml rename to .changes/1.9.0/Under the Hood-20240425-144556.yaml diff --git a/.changes/unreleased/Under the Hood-20240517-143743.yaml b/.changes/1.9.0/Under the Hood-20240517-143743.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240517-143743.yaml rename to .changes/1.9.0/Under the Hood-20240517-143743.yaml diff --git a/.changes/unreleased/Under the Hood-20240614-170858.yaml b/.changes/1.9.0/Under the Hood-20240614-170858.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240614-170858.yaml rename to .changes/1.9.0/Under the Hood-20240614-170858.yaml diff --git a/.changes/unreleased/Under the Hood-20240716-174655.yaml b/.changes/1.9.0/Under the Hood-20240716-174655.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240716-174655.yaml rename to .changes/1.9.0/Under the Hood-20240716-174655.yaml diff --git a/.changes/unreleased/Under the Hood-20240719-125618.yaml b/.changes/1.9.0/Under the Hood-20240719-125618.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240719-125618.yaml rename to .changes/1.9.0/Under the Hood-20240719-125618.yaml diff --git a/.changes/unreleased/Under the Hood-20240722-143114.yaml b/.changes/1.9.0/Under the Hood-20240722-143114.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240722-143114.yaml rename to .changes/1.9.0/Under the Hood-20240722-143114.yaml diff --git a/.changes/unreleased/Under the Hood-20240806-215935.yaml b/.changes/1.9.0/Under the Hood-20240806-215935.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240806-215935.yaml rename to .changes/1.9.0/Under the Hood-20240806-215935.yaml diff --git a/.changes/unreleased/Under the Hood-20240917-181147.yaml b/.changes/1.9.0/Under the Hood-20240917-181147.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240917-181147.yaml rename to .changes/1.9.0/Under the Hood-20240917-181147.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 301a00ea9..599c20195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,71 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-snowflake 1.9.0-b1 - October 01, 2024 + +### Features + +- Support refresh_mode and initialize parameters for dynamic tables ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) +- Add tests for cross-database `cast` macro ([#1009](https://github.com/dbt-labs/dbt-snowflake/issues/1009)) +- Cross-database `date` macro ([#1013](https://github.com/dbt-labs/dbt-snowflake/issues/1013)) +- Replace underscores with hyphens in account IDs to prevent SSL issues ([#1068](https://github.com/dbt-labs/dbt-snowflake/issues/1068)) +- Support JWT Authentication ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) +- Improve run times for large projects by reusing connections by default ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) +- Improve run times when using key pair auth by caching the private key ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) +- Add support for Iceberg table materializations. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) +- Microbatch incremental strategy ([#1182](https://github.com/dbt-labs/dbt-snowflake/issues/1182)) +- Add support for Iceberg table format in Dynamic Tables ([#1183](https://github.com/dbt-labs/dbt-snowflake/issues/1183)) +- Add Iceberg format Incremental Models ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) +- Add support for all on_schema_change incremental model strategies. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) + +### Fixes + +- Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability ([#1048](https://github.com/dbt-labs/dbt-snowflake/issues/1048)) +- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) +- Rename targets for tables and views use fully qualified names ([#1031](https://github.com/dbt-labs/dbt-snowflake/issues/1031)) +- Surface SSO token expiration in logs ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) +- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) +- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) +- Use show ... starts with instead of show ... like in _show_object_metadata ([#1102](https://github.com/dbt-labs/dbt-snowflake/issues/1102)) +- Fix issue where dbt-snowflake attempts to drop database roles during grants sync ([#1151](https://github.com/dbt-labs/dbt-snowflake/issues/1151)) + +### Under the Hood + +- Lazy load agate ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) +- Speedup catalog string comparison by using ilike before equals ([#1035](https://github.com/dbt-labs/dbt-snowflake/issues/1035)) +- Improve memory efficiency of the process_results() override. ([#1053](https://github.com/dbt-labs/dbt-snowflake/issues/1053)) +- Automate all manual integration tests for Dynamic Tables ([#1084](https://github.com/dbt-labs/dbt-snowflake/issues/1084)) +- Add support for experimental record/replay testing. ([#1106](https://github.com/dbt-labs/dbt-snowflake/issues/1106)) +- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-snowflake/issues/1136)) +- Add support for Python 3.12 ([#903](https://github.com/dbt-labs/dbt-snowflake/issues/903)) +- Isolating distribution testing ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) +- Change behavior flag semantics to log iceberg flag warnings.. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) + +### Dependencies + +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#869](https://github.com/dbt-labs/dbt-snowflake/pull/869)) +- Bump actions/upload-artifact from 3 to 4 ([#971](https://github.com/dbt-labs/dbt-snowflake/pull/971)) +- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1006](https://github.com/dbt-labs/dbt-snowflake/pull/1006)) +- Bump actions/download-artifact from 3 to 4 ([#1007](https://github.com/dbt-labs/dbt-snowflake/pull/1007)) +- Bump aurelien-baudet/workflow-dispatch from 2 to 4 ([#1093](https://github.com/dbt-labs/dbt-snowflake/pull/1093)) +- Update twine requirement from ~=4.0 to ~=5.1 ([#1120](https://github.com/dbt-labs/dbt-snowflake/pull/1120)) +- Bump pre-commit from 3.7.0 to 3.7.1 ([#1119](https://github.com/dbt-labs/dbt-snowflake/pull/1119)) +- Update wheel requirement from ~=0.42 to ~=0.43 ([#1121](https://github.com/dbt-labs/dbt-snowflake/pull/1121)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1122](https://github.com/dbt-labs/dbt-snowflake/pull/1122)) +- Update tox requirement from ~=4.11 to ~=4.16 ([#1135](https://github.com/dbt-labs/dbt-snowflake/pull/1135)) + +### Contributors +- [@HenkvanDyk,mikealfare](https://github.com/HenkvanDyk,mikealfare) ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) +- [@McKnight-42](https://github.com/McKnight-42) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851), [#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) +- [@amardatar](https://github.com/amardatar) ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) +- [@dwreeves](https://github.com/dwreeves) ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) +- [@leahwicz](https://github.com/leahwicz) ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) +- [@llam15](https://github.com/llam15) ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) +- [@mikealfare,](https://github.com/mikealfare,) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: -- [1.8](https://github.com/dbt-labs/dbt-snowflake/blob/1.8.latest/CHANGELOG.md) -- [1.7](https://github.com/dbt-labs/dbt-snowflake/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-snowflake/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-snowflake/blob/1.4.latest/CHANGELOG.md) diff --git a/dbt/adapters/snowflake/__version__.py b/dbt/adapters/snowflake/__version__.py index 6698ed64c..a4077fff2 100644 --- a/dbt/adapters/snowflake/__version__.py +++ b/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = "1.9.0a1" +version = "1.9.0b1" From d30670f476660d1e75b875fdd8b224353e8618a7 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:13:33 -0400 Subject: [PATCH 17/41] Update database role tests to avoid colliding with parallel test runs (#1197) * add a unique suffix to the database role to avoid colliding with parallel tests * limit the database role privileges to just the test schema * use the test schema prefix for uniqueness --- tests/functional/auth_tests/test_database_role.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/auth_tests/test_database_role.py b/tests/functional/auth_tests/test_database_role.py index f36555e34..c0f93d7d6 100644 --- a/tests/functional/auth_tests/test_database_role.py +++ b/tests/functional/auth_tests/test_database_role.py @@ -42,18 +42,18 @@ def project_config_update(self): return {"models": {"+grants": {"select": [os.getenv("SNOWFLAKE_TEST_ROLE")]}}} @pytest.fixture(scope="class", autouse=True) - def setup(self, project): + def setup(self, project, prefix): """ Create a database role with access to the model we're about to create. The existence of this database role triggered the bug as dbt-snowflake attempts to revoke it if the user also provides a grants config. """ - role = "BLOCKING_DB_ROLE" + role = f"BLOCKING_DB_ROLE_{prefix}" project.run_sql(f"CREATE DATABASE ROLE {role}") sql = f""" GRANT ALL PRIVILEGES ON FUTURE TABLES - IN DATABASE {project.database} + IN SCHEMA {project.test_schema} TO DATABASE ROLE {role} """ project.run_sql(sql) From 7dea1458ba4a63bafa2f4188768f1059b09dabdd Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:35:08 -0700 Subject: [PATCH 18/41] Fix Dynamic Iceberg Table Required DDL Params (#1201) * Fix base location not rendering without subpath and add tests. Take optional off params that are not optional in dynamic table create DDL. * Add changelog. * Revert changes to external volume * revert changes to catalog optionality. * Tabs. * Fix base_location_subpath generation for dynamic tables. --------- Co-authored-by: VersusFacit --- .../unreleased/Fixes-20241008-122635.yaml | 6 ++ .../snowflake/relation_configs/catalog.py | 8 +- .../macros/relations/dynamic_table/create.sql | 2 +- .../relations/dynamic_table/replace.sql | 2 +- tests/functional/iceberg/models.py | 85 +++++++++++++++++++ tests/functional/iceberg/test_table_basic.py | 72 +++------------- 6 files changed, 111 insertions(+), 64 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241008-122635.yaml create mode 100644 tests/functional/iceberg/models.py diff --git a/.changes/unreleased/Fixes-20241008-122635.yaml b/.changes/unreleased/Fixes-20241008-122635.yaml new file mode 100644 index 000000000..c069283d6 --- /dev/null +++ b/.changes/unreleased/Fixes-20241008-122635.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Dynamic Iceberg table base_location_subpath generation fix. +time: 2024-10-08T12:26:35.521308-07:00 +custom: + Author: versusfacit + Issue: "1200" diff --git a/dbt/adapters/snowflake/relation_configs/catalog.py b/dbt/adapters/snowflake/relation_configs/catalog.py index 09e338635..c8d7de40f 100644 --- a/dbt/adapters/snowflake/relation_configs/catalog.py +++ b/dbt/adapters/snowflake/relation_configs/catalog.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Dict, Optional, TYPE_CHECKING, Set +from typing import Any, Dict, Optional, TYPE_CHECKING, Set, List if TYPE_CHECKING: import agate @@ -82,8 +82,10 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any if external_volume := relation_config.config.extra.get("external_volume"): config_dict["external_volume"] = external_volume - if base_location := relation_config.config.extra.get("base_location_subpath"): - config_dict["base_location"] = base_location + catalog_dirs: List[str] = ["_dbt", relation_config.schema, relation_config.name] + if base_location_subpath := relation_config.config.extra.get("base_location_subpath"): + catalog_dirs.append(base_location_subpath) + config_dict["base_location"] = "/".join(catalog_dirs) return config_dict diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql index 0bd190dcc..4ebcf145b 100644 --- a/dbt/include/snowflake/macros/relations/dynamic_table/create.sql +++ b/dbt/include/snowflake/macros/relations/dynamic_table/create.sql @@ -75,7 +75,7 @@ warehouse = {{ dynamic_table.snowflake_warehouse }} {{ optional('external_volume', dynamic_table.catalog.external_volume) }} {{ optional('catalog', dynamic_table.catalog.name) }} - base_location = {{ dynamic_table.catalog.base_location }} + base_location = '{{ dynamic_table.catalog.base_location }}' {{ optional('refresh_mode', dynamic_table.refresh_mode) }} {{ optional('initialize', dynamic_table.initialize) }} as ( diff --git a/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql b/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql index f9ba1275a..2e7b4566a 100644 --- a/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql +++ b/dbt/include/snowflake/macros/relations/dynamic_table/replace.sql @@ -74,7 +74,7 @@ warehouse = {{ dynamic_table.snowflake_warehouse }} {{ optional('external_volume', dynamic_table.catalog.external_volume) }} {{ optional('catalog', dynamic_table.catalog.name) }} - base_location = {{ dynamic_table.catalog.base_location }} + base_location = '{{ dynamic_table.catalog.base_location }}' {{ optional('refresh_mode', dynamic_table.refresh_mode) }} {{ optional('initialize', dynamic_table.initialize) }} as ( diff --git a/tests/functional/iceberg/models.py b/tests/functional/iceberg/models.py new file mode 100644 index 000000000..6433f74bf --- /dev/null +++ b/tests/functional/iceberg/models.py @@ -0,0 +1,85 @@ +_MODEL_BASIC_TABLE_MODEL = """ +{{ + config( + materialized = "table", + cluster_by=['id'], + ) +}} +select 1 as id +""" + +_MODEL_BASIC_ICEBERG_MODEL = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', + table_format='iceberg', + external_volume='s3_iceberg_snow', +) }} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='1 minute', + refresh_mode='INCREMENTAL', + table_format='iceberg', + external_volume='s3_iceberg_snow', + base_location_subpath='subpath', +) }} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BUILT_ON_ICEBERG_TABLE = """ +{{ + config( + materialized = "table", + ) +}} +select * from {{ ref('iceberg_table') }} +""" + +_MODEL_TABLE_BEFORE_SWAP = """ +{{ + config( + materialized = "table", + ) +}} +select 1 as id +""" + +_MODEL_VIEW_BEFORE_SWAP = """ +select 1 as id +""" + +_MODEL_TABLE_FOR_SWAP_ICEBERG = """ +{{ + config( + materialized = "table", + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_subpath="subpath", + ) +}} +select 1 as id +""" diff --git a/tests/functional/iceberg/test_table_basic.py b/tests/functional/iceberg/test_table_basic.py index 0bfdf59f1..e835a5fce 100644 --- a/tests/functional/iceberg/test_table_basic.py +++ b/tests/functional/iceberg/test_table_basic.py @@ -4,64 +4,16 @@ from dbt.tests.util import run_dbt, rm_file, write_file -_MODEL_BASIC_TABLE_MODEL = """ -{{ - config( - materialized = "table", - cluster_by=['id'], - ) -}} -select 1 as id -""" - -_MODEL_BASIC_ICEBERG_MODEL = """ -{{ - config( - transient = "true", - materialized = "table", - cluster_by=['id'], - table_format="iceberg", - external_volume="s3_iceberg_snow", - base_location_subpath="subpath", - ) -}} - -select * from {{ ref('first_table') }} -""" - -_MODEL_BUILT_ON_ICEBERG_TABLE = """ -{{ - config( - materialized = "table", - ) -}} -select * from {{ ref('iceberg_table') }} -""" - -_MODEL_TABLE_BEFORE_SWAP = """ -{{ - config( - materialized = "table", - ) -}} -select 1 as id -""" - -_MODEL_VIEW_BEFORE_SWAP = """ -select 1 as id -""" - -_MODEL_TABLE_FOR_SWAP_ICEBERG = """ -{{ - config( - materialized = "table", - table_format="iceberg", - external_volume="s3_iceberg_snow", - base_location_subpath="subpath", - ) -}} -select 1 as id -""" +from tests.functional.iceberg.models import ( + _MODEL_BASIC_TABLE_MODEL, + _MODEL_BASIC_ICEBERG_MODEL, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, + _MODEL_BUILT_ON_ICEBERG_TABLE, + _MODEL_TABLE_BEFORE_SWAP, + _MODEL_VIEW_BEFORE_SWAP, + _MODEL_TABLE_FOR_SWAP_ICEBERG, +) class TestIcebergTableBuilds: @@ -75,11 +27,13 @@ def models(self): "first_table.sql": _MODEL_BASIC_TABLE_MODEL, "iceberg_table.sql": _MODEL_BASIC_ICEBERG_MODEL, "table_built_on_iceberg_table.sql": _MODEL_BUILT_ON_ICEBERG_TABLE, + "dynamic_table.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL, + "dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, } def test_iceberg_tables_build_and_can_be_referred(self, project): run_results = run_dbt() - assert len(run_results) == 3 + assert len(run_results) == 5 class TestIcebergTableTypeBuildsOnExistingTable: From 8825ccb8bf2c1343f91a5f3c6ea0c7a05376f1ee Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:16:35 -0400 Subject: [PATCH 19/41] Add query tag back to incremental models (#1212) * add query tag back to incremental models * fix query tag test --- .../macros/materializations/incremental.sql | 1 + tests/functional/query_tag/test_query_tags.py | 34 +++++++------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt/include/snowflake/macros/materializations/incremental.sql index 12645cd11..d73525d6d 100644 --- a/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/dbt/include/snowflake/macros/materializations/incremental.sql @@ -58,6 +58,7 @@ {% materialization incremental, adapter='snowflake', supported_languages=['sql', 'python'] -%} + {% set original_query_tag = set_query_tag() %} {#-- Set vars --#} {%- set full_refresh_mode = (should_full_refresh()) -%} diff --git a/tests/functional/query_tag/test_query_tags.py b/tests/functional/query_tag/test_query_tags.py index 421aae1b6..4ddafabb2 100644 --- a/tests/functional/query_tag/test_query_tags.py +++ b/tests/functional/query_tag/test_query_tags.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.util import run_dbt + snapshots__snapshot_query_tag_sql = """ {% snapshot snapshot_query_tag %} {{ @@ -14,16 +15,15 @@ }} select 1 as id, 'blue' as color {% endsnapshot %} - """ + models__table_model_query_tag_sql = """ {{ config(materialized = 'table') }} - select 1 as id - """ + models__models_config_yml = """ version: 2 @@ -33,23 +33,21 @@ - name: id data_tests: - unique - """ + models__view_model_query_tag_sql = """ {{ config(materialized = 'view') }} - select 1 as id - """ + models__incremental_model_query_tag_sql = """ {{ config(materialized = 'incremental', unique_key = 'id') }} - select 1 as id - """ + macros__check_tag_sql = """ {% macro check_query_tag() %} @@ -61,12 +59,12 @@ {% endif %} {% endmacro %} - """ + seeds__seed_query_tag_csv = """id 1 -""" +""".strip() class TestQueryTag: @@ -95,20 +93,14 @@ def seeds(self): def project_config_update(self, prefix): return { "config-version": 2, - "models": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, - "seeds": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, - "snapshots": { - "tests": {"query_tag": prefix, "post-hook": "{{ check_tag() }}"}, - }, + "models": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "seeds": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, + "snapshots": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}, "tests": {"test": {"query_tag": prefix, "post-hook": "{{ check_query_tag() }}"}}, } def build_all_with_query_tags(self, project, prefix): - run_dbt(["build", "--vars", '{{"check_tag": "{}"}}'.format(prefix)]) + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) def test_snowflake_query_tag(self, project, prefix): self.build_all_with_query_tags(project, prefix) @@ -130,7 +122,7 @@ def profiles_config_update(self, prefix): return {"query_tag": prefix} def build_all_with_query_tags(self, project, prefix): - run_dbt(["build", "--vars", '{{"check_tag": "{}"}}'.format(prefix)]) + run_dbt(["build", "--vars", '{{"query_tag": "{}"}}'.format(prefix)]) def test_snowflake_query_tag(self, project, prefix): self.build_all_with_query_tags(project, prefix) From 78f86674bc726000686a484c4d2f0e2f9d350ed1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:48:05 -0400 Subject: [PATCH 20/41] Drop support for Python 3.8 (#1211) * drop support for python 3.8 --- .../Breaking Changes-20241016-183143.yaml | 6 +++ .github/scripts/integration-test-matrix.js | 4 +- .github/workflows/main.yml | 8 +-- .pre-commit-config.yaml | 1 - CONTRIBUTING.md | 2 +- Makefile | 10 ++-- dev-requirements.txt | 3 +- docker/Dockerfile | 4 +- docker/dev.Dockerfile | 52 +++++++++---------- setup.py | 8 ++- tox.ini | 6 +-- 11 files changed, 53 insertions(+), 51 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-183143.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-183143.yaml b/.changes/unreleased/Breaking Changes-20241016-183143.yaml new file mode 100644 index 000000000..26cc4b6de --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-183143.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:31:43.4167-04:00 +custom: + Author: mikealfare + Issue: "1211" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 81386c54e..756c21d5e 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"]; + const defaultPythonVersion = "3.9"; + const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"]; const supportedAdapters = ["snowflake"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95ce18033..4913917f4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -71,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] env: TOXENV: "unit" @@ -127,7 +127,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -175,7 +175,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e14455e28..807e32a9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45e0054c5..5b68aa03a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ WARNING: The parameters in your `test.env` file must link to a valid Snowflake a There are a few methods for running tests locally. #### `tox` -`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.8, Python 3.9, Python 3.10, and `flake8` in parallel. Run `tox -e py38` to invoke tests on Python version 3.8 only (use py38, py39, or py310). Tox recipes are found in `tox.ini`. +`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.9 and Python 3.10, and `flake8` in parallel. Run `tox -e py39` to invoke tests on Python version 3.9 only (use py39 or py310). Tox recipes are found in `tox.ini`. #### `pytest` You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so: diff --git a/Makefile b/Makefile index b42de9147..9a2376f47 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,20 @@ dev-uninstall: ## Uninstalls all packages while maintaining the virtual environm pip uninstall -y dbt-snowflake .PHONY: unit -unit: ## Runs unit tests with py38. +unit: ## Runs unit tests with py39. @\ - tox -e py38 + tox -e py39 .PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. +test: ## Runs unit tests with py39 and code checks against staged changes. @\ - tox -p -e py38; \ + tox -p -e py39; \ pre-commit run --all-files .PHONY: integration integration: ## Runs snowflake integration tests with py38. @\ - tox -e py38-snowflake -- + tox -e py39-snowflake -- .PHONY: clean @echo "cleaning repo" diff --git a/dev-requirements.txt b/dev-requirements.txt index f3d120eec..906003768 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,8 +6,7 @@ git+https://github.com/dbt-labs/dbt-common.git # dev ipdb~=0.13.13 -pre-commit~=3.7.0;python_version>="3.9" -pre-commit~=3.5.0;python_version<"3.9" +pre-commit~=3.7.0 # test ddtrace==2.3.0 diff --git a/docker/Dockerfile b/docker/Dockerfile index d256dcac4..17315b12d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # this image gets published to GHCR for production use ARG py_version=3.11.2 -FROM python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -25,7 +25,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir -FROM base as dbt-snowflake +FROM base AS dbt-snowflake ARG commit_ref=main diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index 0fc667048..44f86f005 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -1,43 +1,43 @@ # this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM ubuntu:22.04 as base +FROM ubuntu:24.04 AS base # prevent python installation from asking for time zone region ARG DEBIAN_FRONTEND=noninteractive # add python repository RUN apt-get update \ - && apt-get install -y software-properties-common=0.99.22.9 \ - && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y software-properties-common=0.99.48 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # install python RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential=12.9ubuntu3 \ - git-all=1:2.34.1-1ubuntu1.10 \ - python3.8=3.8.19-1+jammy1 \ - python3.8-dev=3.8.19-1+jammy1 \ - python3.8-distutils=3.8.19-1+jammy1 \ - python3.8-venv=3.8.19-1+jammy1 \ - python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - python3-wheel=0.37.1-2ubuntu0.22.04.1 \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y --no-install-recommends \ + build-essential=12.10ubuntu1 \ + git-all=1:2.43.0-1ubuntu7.1 \ + python3.9=3.9.20-1+noble1 \ + python3.9-dev=3.9.20-1+noble1 \ + python3.9-distutils=3.9.20-1+noble1 \ + python3.9-venv=3.9.20-1+noble1 \ + python3-pip=24.0+dfsg-1ubuntu1 \ + python3-wheel=0.42.0-2 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ # update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 -FROM base as dbt-snowflake-dev +FROM base AS dbt-snowflake-dev -HEALTHCHECK CMD python3 --version || exit 1 +HEALTHCHECK CMD python --version || exit 1 # send stdout/stderr to terminal ENV PYTHONUNBUFFERED=1 @@ -47,4 +47,4 @@ WORKDIR /opt/code VOLUME /opt/code # create a virtual environment -RUN python3 -m venv /opt/venv +RUN python -m venv /opt/venv diff --git a/setup.py b/setup.py index f542b9fcb..c0716341d 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ import sys # require python 3.8 or newer -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) @@ -72,12 +72,10 @@ def _plugin_version() -> str: "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tox.ini b/tox.ini index d6f040a61..f6952efaf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311,py312 +envlist = py39,py310,py311,py312 -[testenv:{unit,py38,py39,py310,py311,py312,py}] +[testenv:{unit,py39,py310,py311,py312,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py312,py}-{snowflake}] +[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}] description = adapter plugin integration testing skip_install = true passenv = From ea33cbec035c32052008c07c90ef799e024aa7dc Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:08:10 -0700 Subject: [PATCH 21/41] Add url. (#1222) --- dbt/include/snowflake/macros/relations/table/create.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index e60b93039..e2141df4d 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,7 +1,7 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} - {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to .') %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs.') %} {%- endif %} {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} From df643a0ed4bec6e0e961594d8171eb2c808d249b Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:20:45 -0700 Subject: [PATCH 22/41] Add snowflake telemetry. (#1209) * Add snowflake telemetry. * Add changelog. * Temporary dev branch switch. * Correct version import * bump ci * Temporary dev branch switch. * Temporary dev branch switch take 2. * Alter to meet new base schema changes. * Fix input args. * Change field name. --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../Under the Hood-20241016-035544.yaml | 6 +++++ dbt/adapters/snowflake/impl.py | 17 ++++++++++++ dev-requirements.txt | 2 +- setup.py | 2 +- tests/unit/test_adapter_telemetry.py | 27 +++++++++++++++++++ 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20241016-035544.yaml create mode 100644 tests/unit/test_adapter_telemetry.py diff --git a/.changes/unreleased/Under the Hood-20241016-035544.yaml b/.changes/unreleased/Under the Hood-20241016-035544.yaml new file mode 100644 index 000000000..59e4f70de --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241016-035544.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add telemetry function +time: 2024-10-16T03:55:44.144174-07:00 +custom: + Author: versusfacit + Issue: "301" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 6320893e1..89c21f531 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -4,6 +4,7 @@ from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability +from dbt.adapters.contracts.relation import RelationConfig from dbt.adapters.sql import SQLAdapter from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -25,6 +26,7 @@ SnowflakeRelationType, TableFormat, ) + from dbt.adapters.snowflake import SnowflakeColumn from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation @@ -419,3 +421,18 @@ def valid_incremental_strategies(self): def debug_query(self): """Override for DebugTask method""" self.execute("select 1 as id") + + @classmethod + def _get_adapter_specific_run_info(cls, config: RelationConfig) -> Dict[str, Any]: + table_format: Optional[str] = None + if ( + config + and hasattr(config, "_extra") + and (relation_format := config._extra.get("table_format")) + ): + table_format = relation_format + + return { + "adapter_type": "snowflake", + "table_format": table_format, + } diff --git a/dev-requirements.txt b/dev-requirements.txt index 906003768..0ceb365db 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index c0716341d..fcea43309 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", diff --git a/tests/unit/test_adapter_telemetry.py b/tests/unit/test_adapter_telemetry.py new file mode 100644 index 000000000..498676b77 --- /dev/null +++ b/tests/unit/test_adapter_telemetry.py @@ -0,0 +1,27 @@ +from unittest import mock + +import dbt.adapters.snowflake.__version__ + +from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.base.relation import AdapterTrackingRelationInfo + + +def test_telemetry_with_snowflake_details(): + mock_model_config = mock.MagicMock() + mock_model_config._extra = mock.MagicMock() + mock_model_config._extra = { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + res = SnowflakeAdapter.get_adapter_run_info(mock_model_config) + + assert res.adapter_name == "snowflake" + assert res.base_adapter_version == dbt.adapters.__about__.version + assert res.adapter_version == dbt.adapters.snowflake.__version__.version + assert res.model_adapter_details == { + "adapter_type": "snowflake", + "table_format": "iceberg", + } + + assert type(res) is AdapterTrackingRelationInfo From ad4ccadb0411826bd3053800431bbc7cfc9a78c2 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 29 Oct 2024 07:50:49 -0700 Subject: [PATCH 23/41] Revert requirements. (#1225) Co-authored-by: VersusFacit --- dev-requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0ceb365db..906003768 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry +git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index fcea43309..c0716341d 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@ADAP-301/add-adapter-telemetry", + "dbt-adapters>=1.7,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 1d299923e34c96f2e96a5215ac196658f86ce1d1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:35:37 -0400 Subject: [PATCH 24/41] Add test cases for generic tests config (#1172) * add test cases for generic tests config * update tests --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Colin --- .../functional/generic_test_tests/__init__.py | 0 tests/functional/generic_test_tests/_files.py | 90 +++++++++++++++++++ .../functional/generic_test_tests/_models.py | 0 .../functional/generic_test_tests/_schemas.py | 0 .../generic_test_tests/test_generic_tests.py | 54 +++++++++++ 5 files changed, 144 insertions(+) create mode 100644 tests/functional/generic_test_tests/__init__.py create mode 100644 tests/functional/generic_test_tests/_files.py create mode 100644 tests/functional/generic_test_tests/_models.py create mode 100644 tests/functional/generic_test_tests/_schemas.py create mode 100644 tests/functional/generic_test_tests/test_generic_tests.py diff --git a/tests/functional/generic_test_tests/__init__.py b/tests/functional/generic_test_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/_files.py b/tests/functional/generic_test_tests/_files.py new file mode 100644 index 000000000..a9743e43e --- /dev/null +++ b/tests/functional/generic_test_tests/_files.py @@ -0,0 +1,90 @@ +SCHEMA__CONTROL = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null +""" + + +SCHEMA__EXPLICIT_WAREHOUSE = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__NOT_NULL = """ +version: 2 +models: + - name: facts + columns: + - name: value + data_tests: + - not_null: + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__RELATIONSHIPS = """ +version: 2 +models: + - name: facts + columns: + - name: color + data_tests: + - relationships: + to: ref('my_colors') + field: color + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SCHEMA__ACCEPTED_VALUES = """ +version: 2 +models: + - name: colors + columns: + - name: color + data_tests: + - accepted_values: + values: ['blue', 'red', 'green'] + config: + snowflake_warehouse: DBT_TESTING_ALT +""" + + +SEED__COLORS = """ +color +blue +green +red +yellow +""".strip() + + +# record 10 is missing a value +# record 7 has a color that's not on COLORS +SEED__FACTS = """ +id,color,value +1,blue,10 +2,red,20 +3,green,30 +4,yellow,40 +5,blue,50 +6,red,60 +7,orange,70 +8,green,80 +9,yellow,90 +10,blue, +""".strip() diff --git a/tests/functional/generic_test_tests/_models.py b/tests/functional/generic_test_tests/_models.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/_schemas.py b/tests/functional/generic_test_tests/_schemas.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/generic_test_tests/test_generic_tests.py b/tests/functional/generic_test_tests/test_generic_tests.py new file mode 100644 index 000000000..a653a363b --- /dev/null +++ b/tests/functional/generic_test_tests/test_generic_tests.py @@ -0,0 +1,54 @@ +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + +from tests.functional.generic_test_tests import _files + + +class TestWarehouseConfig: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "colors.csv": _files.SEED__COLORS, + "facts.csv": _files.SEED__FACTS, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield + + +class TestWarehouseConfigControl(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__CONTROL} + + def test_expected_warehouse(self, project): + results, logs = run_dbt_and_capture(["test"]) + assert len(results) == 1 + + +class TestWarehouseConfigExplicitWarehouse(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__EXPLICIT_WAREHOUSE} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"]) + assert "use warehouse " in logs + + +class TestWarehouseConfigNotNull(TestWarehouseConfig): + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": _files.SCHEMA__NOT_NULL} + + def test_expected_warehouse(self, project): + _, logs = run_dbt_and_capture(["test", "--log-level", "debug"], expect_pass=False) + assert "use warehouse " in logs From a16654bed4d67d1a922bf87e579a285c35bd55ac Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:57:52 -0800 Subject: [PATCH 25/41] move github runner from macos-12 to macos-14 (#1230) * move github runner from macos-12 to macos-14 --- .github/scripts/integration-test-matrix.js | 4 ++-- .github/workflows/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 756c21d5e..e2c88b00b 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -44,7 +44,7 @@ module.exports = ({ context }) => { if (labels.includes("test macos") || testAllLabel) { include.push({ - os: "macos-12", + os: "macos-14", adapter, "python-version": pythonVersion, }); @@ -78,7 +78,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-12"]) { + for (const operatingSystem of ["windows-latest", "macos-14"]) { include.push({ os: operatingSystem, adapter: adapter, diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4913917f4..24d2fa60b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,7 +174,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-latest, macos-14, windows-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] From efeb82b087c95662a590ecf96e2c8cdf62cf68ed Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 6 Nov 2024 12:06:31 -0800 Subject: [PATCH 26/41] remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base (#1233) * remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base * remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base --- .../unreleased/Under the Hood-20241106-113249.yaml | 6 ++++++ dbt/adapters/snowflake/connections.py | 13 ++++--------- tests/unit/test_snowflake_adapter.py | 3 ++- 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20241106-113249.yaml diff --git a/.changes/unreleased/Under the Hood-20241106-113249.yaml b/.changes/unreleased/Under the Hood-20241106-113249.yaml new file mode 100644 index 000000000..0437a8c88 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241106-113249.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base +time: 2024-11-06T11:32:49.503467-08:00 +custom: + Author: colin-rogers-dbt + Issue: "1233" diff --git a/dbt/adapters/snowflake/connections.py b/dbt/adapters/snowflake/connections.py index 10bee30f0..fc2c09c19 100644 --- a/dbt/adapters/snowflake/connections.py +++ b/dbt/adapters/snowflake/connections.py @@ -84,11 +84,6 @@ def snowflake_private_key(private_key: RSAPrivateKey) -> bytes: ) -@dataclass -class SnowflakeAdapterResponse(AdapterResponse): - query_id: str = "" - - @dataclass class SnowflakeCredentials(Credentials): account: str @@ -447,17 +442,17 @@ def cancel(self, connection): logger.debug("Cancel query '{}': {}".format(connection_name, res)) @classmethod - def get_response(cls, cursor) -> SnowflakeAdapterResponse: + def get_response(cls, cursor) -> AdapterResponse: code = cursor.sqlstate if code is None: code = "SUCCESS" - - return SnowflakeAdapterResponse( + query_id = str(cursor.sfqid) if cursor.sfqid is not None else None + return AdapterResponse( _message="{} {}".format(code, cursor.rowcount), rows_affected=cursor.rowcount, code=code, - query_id=cursor.sfqid, + query_id=query_id, ) # disable transactional logic by default on Snowflake diff --git a/tests/unit/test_snowflake_adapter.py b/tests/unit/test_snowflake_adapter.py index 32e73eb45..aa580aad2 100644 --- a/tests/unit/test_snowflake_adapter.py +++ b/tests/unit/test_snowflake_adapter.py @@ -60,8 +60,10 @@ def setUp(self): self.handle = mock.MagicMock(spec=snowflake_connector.SnowflakeConnection) self.cursor = self.handle.cursor.return_value self.mock_execute = self.cursor.execute + self.mock_execute.return_value = mock.MagicMock(sfqid="42") self.patcher = mock.patch("dbt.adapters.snowflake.connections.snowflake.connector.connect") self.snowflake = self.patcher.start() + self.snowflake.connect.cursor.return_value = mock.MagicMock(sfqid="42") # Create the Manifest.state_check patcher @mock.patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") @@ -90,7 +92,6 @@ def _mock_state_check(self): self.qh_patch = mock.patch.object(self.adapter.connections.query_header, "add") self.mock_query_header_add = self.qh_patch.start() self.mock_query_header_add.side_effect = lambda q: "/* dbt */\n{}".format(q) - self.adapter.acquire_connection() inject_adapter(self.adapter, SnowflakePlugin) From 01dbd70ba3959fab049d83fee93b93ceeb2ec013 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 6 Nov 2024 17:33:12 -0500 Subject: [PATCH 27/41] [Microbatch] Optimizations: use view for temp relation + remove `using` clause during delete statement (#1192) --- .changes/unreleased/Fixes-20241104-104610.yaml | 7 +++++++ .../snowflake/macros/materializations/incremental.sql | 8 ++++---- dbt/include/snowflake/macros/materializations/merge.sql | 1 - 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241104-104610.yaml diff --git a/.changes/unreleased/Fixes-20241104-104610.yaml b/.changes/unreleased/Fixes-20241104-104610.yaml new file mode 100644 index 000000000..c512d0bdd --- /dev/null +++ b/.changes/unreleased/Fixes-20241104-104610.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: 'Performance fixes for snowflake microbatch strategy: use temp view instead + of table, remove unnecessary ''using'' clause' +time: 2024-11-04T10:46:10.005317-05:00 +custom: + Author: michelleark + Issue: "1228" diff --git a/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt/include/snowflake/macros/materializations/incremental.sql index d73525d6d..dbb79de02 100644 --- a/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/dbt/include/snowflake/macros/materializations/incremental.sql @@ -20,7 +20,7 @@ The append strategy can use a view because it will run a single INSERT statement. - When unique_key is none, the delete+insert strategy can use a view beacuse a + When unique_key is none, the delete+insert and microbatch strategies can use a view beacuse a single INSERT statement is run with no DELETES as part of the statement. Otherwise, play it safe by using a temporary table. #} */ @@ -32,10 +32,10 @@ ) %} {% endif %} - {% if strategy == "delete+insert" and tmp_relation_type is not none and tmp_relation_type != "table" and unique_key is not none %} + {% if strategy in ["delete+insert", "microbatch"] and tmp_relation_type is not none and tmp_relation_type != "table" and unique_key is not none %} {% do exceptions.raise_compiler_error( "In order to maintain consistent results when `unique_key` is not none, - the `delete+insert` strategy only supports `table` for `tmp_relation_type` but " + the `" ~ strategy ~ "` strategy only supports `table` for `tmp_relation_type` but " ~ tmp_relation_type ~ " was specified." ) %} @@ -49,7 +49,7 @@ {{ return("view") }} {% elif strategy in ("default", "merge", "append") %} {{ return("view") }} - {% elif strategy == "delete+insert" and unique_key is none %} + {% elif strategy in ["delete+insert", "microbatch"] and unique_key is none %} {{ return("view") }} {% else %} {{ return("table") }} diff --git a/dbt/include/snowflake/macros/materializations/merge.sql b/dbt/include/snowflake/macros/materializations/merge.sql index 57c58afdd..c8ac8d6fd 100644 --- a/dbt/include/snowflake/macros/materializations/merge.sql +++ b/dbt/include/snowflake/macros/materializations/merge.sql @@ -66,7 +66,6 @@ {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} delete from {{ target }} DBT_INTERNAL_TARGET - using {{ source }} where ( {% for predicate in incremental_predicates %} {%- if not loop.first %}and {% endif -%} {{ predicate }} From f6468f60a6eafff584799122e34608aabb7414e4 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Sat, 9 Nov 2024 17:41:44 -0800 Subject: [PATCH 28/41] Normalize is_iceberg Field for Consistency in Snowflake with QUOTED_IDENTIFIERS_IGNORE_CASE (#1229) * Reproduce. Test fail. Fix. Test green. * Add changelog. * Make a test that tests the actual behavior without hardcoding the quoted flag. * I can't believe how elegant a solution this is. Use a session-only configuration. * Fix test. * Simplify logic for this by moving this away from a quoting situation. Columns that unquoted are capitalized regardless of the quoting ignore flag. So we let it be uppercase and then normalize it down to lowercase in Python. * edit comments. --------- Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Fixes-20241104-172349.yaml | 6 ++ dbt/adapters/snowflake/impl.py | 5 ++ dbt/include/snowflake/macros/adapters.sql | 3 +- .../list_relations_tests/test_show_objects.py | 60 +++++++++++++++---- 4 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241104-172349.yaml diff --git a/.changes/unreleased/Fixes-20241104-172349.yaml b/.changes/unreleased/Fixes-20241104-172349.yaml new file mode 100644 index 000000000..07c90d93c --- /dev/null +++ b/.changes/unreleased/Fixes-20241104-172349.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Iceberg quoting ignore fix. +time: 2024-11-04T17:23:49.706297-08:00 +custom: + Author: versusfacit + Issue: "1227" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 89c21f531..dc0926d93 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -263,6 +263,11 @@ def list_relations_without_caching( # this can be collapsed once Snowflake adds is_iceberg to show objects columns = ["database_name", "schema_name", "name", "kind", "is_dynamic"] if self.behavior.enable_iceberg_materializations.no_warn: + # The QUOTED_IDENTIFIERS_IGNORE_CASE setting impacts column names like + # is_iceberg which is created by dbt, but it does not affect the case + # of column values in Snowflake's SHOW OBJECTS query! This + # normalization step ensures metadata queries are handled consistently. + schema_objects = schema_objects.rename(column_names={"IS_ICEBERG": "is_iceberg"}) columns.append("is_iceberg") return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)] diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index b60cea0b0..0ca756c6c 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -135,7 +135,6 @@ {% endmacro %} {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} - {%- set max_total_results = max_results_per_iter * max_iter -%} {%- set sql -%} {% if schema_relation is string %} @@ -147,7 +146,7 @@ {# -- Gated for performance reason. If you don't want Iceberg, you shouldn't pay the -- latency penalty. #} {% if adapter.behavior.enable_iceberg_materializations.no_warn %} - select all_objects.*, is_iceberg as "is_iceberg" + select all_objects.*, is_iceberg from table(result_scan(last_query_id(-1))) all_objects left join INFORMATION_SCHEMA.tables as all_tables on all_tables.table_name = all_objects."name" diff --git a/tests/functional/adapter/list_relations_tests/test_show_objects.py b/tests/functional/adapter/list_relations_tests/test_show_objects.py index e5eee39d9..91fb94f79 100644 --- a/tests/functional/adapter/list_relations_tests/test_show_objects.py +++ b/tests/functional/adapter/list_relations_tests/test_show_objects.py @@ -3,6 +3,8 @@ import pytest +from pathlib import Path + from dbt.adapters.factory import get_adapter_by_type from dbt.adapters.snowflake import SnowflakeRelation @@ -41,8 +43,32 @@ """ ) +_MODEL_ICEBERG = """ +{{ + config( + materialized = "table", + table_format="iceberg", + external_volume="s3_iceberg_snow", + ) +}} + +select 1 +""" + + +class ShowObjectsBase: + @staticmethod + def list_relations_without_caching(project) -> List[SnowflakeRelation]: + my_adapter = get_adapter_by_type("snowflake") + schema = my_adapter.Relation.create( + database=project.database, schema=project.test_schema, identifier="" + ) + with get_connection(my_adapter): + relations = my_adapter.list_relations_without_caching(schema) + return relations + -class TestShowObjects: +class TestShowObjects(ShowObjectsBase): views: int = 10 tables: int = 10 dynamic_tables: int = 10 @@ -66,16 +92,6 @@ def setup(self, project): run_dbt(["seed"]) run_dbt(["run"]) - @staticmethod - def list_relations_without_caching(project) -> List[SnowflakeRelation]: - my_adapter = get_adapter_by_type("snowflake") - schema = my_adapter.Relation.create( - database=project.database, schema=project.test_schema, identifier="" - ) - with get_connection(my_adapter): - relations = my_adapter.list_relations_without_caching(schema) - return relations - def test_list_relations_without_caching(self, project): relations = self.list_relations_without_caching(project) assert len([relation for relation in relations if relation.is_view]) == self.views @@ -87,3 +103,25 @@ def test_list_relations_without_caching(self, project): len([relation for relation in relations if relation.is_dynamic_table]) == self.dynamic_tables ) + + +class TestShowIcebergObjects(ShowObjectsBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": _MODEL_ICEBERG} + + def test_quoting_ignore_flag_doesnt_break_iceberg_metadata(self, project): + """https://github.com/dbt-labs/dbt-snowflake/issues/1227 + + The list relations function involves a metadata sub-query. Regardless of + QUOTED_IDENTIFIERS_IGNORE_CASE, this function will fail without proper + normalization within the encapsulating python function after the macro invocation + returns. This test verifies that normalization is working. + """ + run_dbt(["run"]) + + self.list_relations_without_caching(project) From 7c63a40c6948e0bd27d756e7adb9604b44347352 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:51:51 -0500 Subject: [PATCH 29/41] Make `list_relations_without_caching` pagination configurable (#1235) * make list relations configurable * update iteration to page in the config settings * update the warning to recommend how to account for breaching the limit on list_relations --- .../unreleased/Features-20241107-170307.yaml | 7 + dbt/include/snowflake/macros/adapters.sql | 10 +- .../list_relations_tests/test_pagination.py | 247 +++++------------- 3 files changed, 85 insertions(+), 179 deletions(-) create mode 100644 .changes/unreleased/Features-20241107-170307.yaml diff --git a/.changes/unreleased/Features-20241107-170307.yaml b/.changes/unreleased/Features-20241107-170307.yaml new file mode 100644 index 000000000..1479c5805 --- /dev/null +++ b/.changes/unreleased/Features-20241107-170307.yaml @@ -0,0 +1,7 @@ +kind: Features +body: 'Allow configurable pagination on list_relations_without_caching to support + users with a large number of objects per schema' +time: 2024-11-07T17:03:07.826352-05:00 +custom: + Author: mikealfare + Issue: "1234" diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index 0ca756c6c..3c93d41ad 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -111,9 +111,10 @@ {%- if loop.index == max_iter -%} {%- set msg -%} - dbt will list a maximum of {{ max_total_results }} objects in schema {{ schema_relation }}. - Your schema exceeds this limit. Please contact support@getdbt.com for troubleshooting tips, - or review and reduce the number of objects contained. + dbt is currently configured to list a maximum of {{ max_total_results }} objects per schema. + {{ schema_relation }} exceeds this limit. If this is expected, you may configure this limit + by setting list_relations_per_page and list_relations_page_limit in your project flags. + It is recommended to start by increasing list_relations_page_limit to something more than the default of 10. {%- endset -%} {% do exceptions.raise_compiler_error(msg) %} @@ -135,6 +136,9 @@ {% endmacro %} {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} + + {%- set max_results_per_iter = adapter.config.flags.get('list_relations_per_page', max_results_per_iter) -%} + {%- set max_iter = adapter.config.flags.get('list_relations_page_limit', max_iter) -%} {%- set max_total_results = max_results_per_iter * max_iter -%} {%- set sql -%} {% if schema_relation is string %} diff --git a/tests/functional/adapter/list_relations_tests/test_pagination.py b/tests/functional/adapter/list_relations_tests/test_pagination.py index 407f9c501..7dd382af5 100644 --- a/tests/functional/adapter/list_relations_tests/test_pagination.py +++ b/tests/functional/adapter/list_relations_tests/test_pagination.py @@ -1,34 +1,31 @@ import os + import pytest -import json -from dbt.tests.util import run_dbt, run_dbt_and_capture -from dbt.adapters.snowflake import SnowflakeRelation # Ensure this is the correct import path - -# Testing rationale: -# - snowflake SHOW TERSE OBJECTS command returns at max 10K objects in a single call -# - when dbt attempts to write into a schema with more than 10K objects, compilation will fail -# unless we paginate the result -# - however, testing this process is difficult at a full scale of 10K actual objects populated -# into a fresh testing schema -# - accordingly, we create a smaller set of views and test the looping iteration logic in -# smaller chunks - -NUM_VIEWS = 90 -NUM_DYNAMIC_TABLES = 10 -# the total number should be between the numbers referenced in the "passing" and "failing" macros below -# - MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING (11 iter * 10 results per iter -> 110 objects) -# - MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR (33 iter * 3 results per iter -> 99 objects) -NUM_EXPECTED_RELATIONS = 1 + NUM_VIEWS + NUM_DYNAMIC_TABLES - -TABLE_BASE_SQL = """ -{{ config(materialized='table') }} +from dbt_common.exceptions import CompilationError +from dbt.tests.util import run_dbt + +""" +Testing rationale: +- snowflake SHOW TERSE OBJECTS command returns at max 10K objects in a single call +- when dbt attempts to write into a schema with more than 10K objects, compilation will fail + unless we paginate the result +- we default pagination to 10 pages, but users want to configure this + - we instead use that here to force failures by making it smaller +""" + + +TABLE = """ +{{ config(materialized='table') }} select 1 as id -""".lstrip() +""" + -VIEW_X_SQL = """ +VIEW = """ +{{ config(materialized='view') }} select id from {{ ref('my_model_base') }} -""".lstrip() +""" + DYNAMIC_TABLE = ( """ @@ -44,173 +41,71 @@ """ ) -MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING = """ -{% macro validate_list_relations_without_caching(schema_relation) %} - {% set relation_list_result = snowflake__list_relations_without_caching(schema_relation, max_iter=11, max_results_per_iter=10) %} - {% set n_relations = relation_list_result | length %} - {{ log("n_relations: " ~ n_relations) }} -{% endmacro %} -""" - -MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR = """ -{% macro validate_list_relations_without_caching_raise_error(schema_relation) %} - {{ snowflake__list_relations_without_caching(schema_relation, max_iter=33, max_results_per_iter=3) }} -{% endmacro %} -""" - - -def parse_json_logs(json_log_output): - parsed_logs = [] - for line in json_log_output.split("\n"): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - return parsed_logs +class BaseConfig: + VIEWS = 90 + DYNAMIC_TABLES = 10 -def find_result_in_parsed_logs(parsed_logs, result_name): - return next( - ( - item["data"]["msg"] - for item in parsed_logs - if result_name in item["data"].get("msg", "msg") - ), - False, - ) - - -def find_exc_info_in_parsed_logs(parsed_logs, exc_info_name): - return next( - ( - item["data"]["exc_info"] - for item in parsed_logs - if exc_info_name in item["data"].get("exc_info", "exc_info") - ), - False, - ) - - -class TestListRelationsWithoutCachingSingle: @pytest.fixture(scope="class") def models(self): - my_models = {"my_model_base.sql": TABLE_BASE_SQL} - for view in range(0, NUM_VIEWS): - my_models.update({f"my_model_{view}.sql": VIEW_X_SQL}) - for dynamic_table in range(0, NUM_DYNAMIC_TABLES): - my_models.update({f"my_dynamic_table_{dynamic_table}.sql": DYNAMIC_TABLE}) + my_models = {"my_model_base.sql": TABLE} + for view in range(0, self.VIEWS): + my_models[f"my_model_{view}.sql"] = VIEW + for dynamic_table in range(0, self.DYNAMIC_TABLES): + my_models[f"my_dynamic_table_{dynamic_table}.sql"] = DYNAMIC_TABLE return my_models - @pytest.fixture(scope="class") - def macros(self): - return { - "validate_list_relations_without_caching.sql": MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING, - } + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["run"]) - def test__snowflake__list_relations_without_caching_termination(self, project): - """ - validates that we do NOT trigger pagination logic snowflake__list_relations_without_caching - macro when there are fewer than max_results_per_iter relations in the target schema - """ - run_dbt(["run", "-s", "my_model_base"]) - - database = project.database - schemas = project.created_schemas - - for schema in schemas: - schema_relation = SnowflakeRelation.create(database=database, schema=schema) - kwargs = {"schema_relation": schema_relation.render()} - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_list_relations_without_caching", - "--args", - str(kwargs), - ] + def test_list_relations(self, project): + kwargs = {"schema_relation": project.test_schema} + with project.adapter.connection_named("__test"): + relations = project.adapter.execute_macro( + "snowflake__list_relations_without_caching", kwargs=kwargs ) + assert len(relations) == self.VIEWS + self.DYNAMIC_TABLES + 1 - parsed_logs = parse_json_logs(log_output) - n_relations = find_result_in_parsed_logs(parsed_logs, "n_relations") - assert n_relations == "n_relations: 1" +class TestListRelationsWithoutCachingSmall(BaseConfig): + pass -class TestListRelationsWithoutCachingFull: - @pytest.fixture(scope="class") - def models(self): - my_models = {"my_model_base.sql": TABLE_BASE_SQL} - for view in range(0, NUM_VIEWS): - my_models.update({f"my_model_{view}.sql": VIEW_X_SQL}) - for dynamic_table in range(0, NUM_DYNAMIC_TABLES): - my_models.update({f"my_dynamic_table_{dynamic_table}.sql": DYNAMIC_TABLE}) - return my_models +class TestListRelationsWithoutCachingLarge(BaseConfig): @pytest.fixture(scope="class") - def macros(self): + def profiles_config_update(self): return { - "validate_list_relations_without_caching.sql": MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING, - "validate_list_relations_without_caching_raise_error.sql": MACROS__VALIDATE__SNOWFLAKE__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR, + "flags": { + "list_relations_per_page": 10, + "list_relations_page_limit": 20, + } } - def test__snowflake__list_relations_without_caching(self, project): - """ - validates pagination logic in snowflake__list_relations_without_caching macro counts - the correct number of objects in the target schema when having to make multiple looped - calls of SHOW TERSE OBJECTS. - """ - # purpose of the first run is to create the replicated views in the target schema - run_dbt(["run"]) - database = project.database - schemas = project.created_schemas - - for schema in schemas: - schema_relation = SnowflakeRelation.create(database=database, schema=schema) - kwargs = {"schema_relation": schema_relation.render()} - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_list_relations_without_caching", - "--args", - str(kwargs), - ] - ) - parsed_logs = parse_json_logs(log_output) - n_relations = find_result_in_parsed_logs(parsed_logs, "n_relations") +class TestListRelationsWithoutCachingTooLarge(BaseConfig): - assert n_relations == f"n_relations: {NUM_EXPECTED_RELATIONS}" - - def test__snowflake__list_relations_without_caching_raise_error(self, project): - """ - validates pagination logic terminates and raises a compilation error - when exceeding the limit of how many results to return. - """ - run_dbt(["run"]) + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "list_relations_per_page": 10, + "list_relations_page_limit": 5, + } + } - database = project.database - schemas = project.created_schemas - - for schema in schemas: - schema_relation = SnowflakeRelation.create(database=database, schema=schema) - - kwargs = {"schema_relation": schema_relation.render()} - _, log_output = run_dbt_and_capture( - [ - "--debug", - "--log-format=json", - "run-operation", - "validate_list_relations_without_caching_raise_error", - "--args", - str(kwargs), - ], - expect_pass=False, - ) - parsed_logs = parse_json_logs(log_output) - traceback = find_exc_info_in_parsed_logs(parsed_logs, "Traceback") - assert "dbt will list a maximum of 99 objects in schema " in traceback + def test_list_relations(self, project): + kwargs = {"schema_relation": project.test_schema} + with project.adapter.connection_named("__test"): + with pytest.raises(CompilationError) as error: + project.adapter.execute_macro( + "snowflake__list_relations_without_caching", kwargs=kwargs + ) + assert "list_relations_per_page" in error.value.msg + assert "list_relations_page_limit" in error.value.msg + + def test_on_run(self, project): + with pytest.raises(CompilationError) as error: + run_dbt(["run"]) + assert "list_relations_per_page" in error.value.msg + assert "list_relations_page_limit" in error.value.msg From dca565bb01a2fc8fa31eb37db0a39847af586270 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:14:20 -0800 Subject: [PATCH 30/41] Adap 209/update iceberg docs note (#1253) * Add url. * Add docs url to flag itself. * Update url in other error for more precision. * Add changelog. --- .changes/unreleased/Under the Hood-20241118-231637.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 1 + dbt/include/snowflake/macros/relations/table/create.sql | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20241118-231637.yaml diff --git a/.changes/unreleased/Under the Hood-20241118-231637.yaml b/.changes/unreleased/Under the Hood-20241118-231637.yaml new file mode 100644 index 000000000..f787a2a79 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241118-231637.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add iceberg docs url to behavior flag. +time: 2024-11-18T23:16:37.926576-08:00 +custom: + Author: versusfacit + Issue: "210" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index dc0926d93..dc256c1cb 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -92,6 +92,7 @@ def _behavior_flags(self) -> List[BehaviorFlag]: "benefits only those actively using it, we've made this behavior opt-in to " "prevent unnecessary latency for other users." ), + "docs_url": "https://docs.getdbt.com/reference/resource-configs/snowflake-configs#iceberg-table-format", } ] diff --git a/dbt/include/snowflake/macros/relations/table/create.sql b/dbt/include/snowflake/macros/relations/table/create.sql index e2141df4d..50bedd78f 100644 --- a/dbt/include/snowflake/macros/relations/table/create.sql +++ b/dbt/include/snowflake/macros/relations/table/create.sql @@ -1,7 +1,7 @@ {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %} - {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs.') %} + {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs#iceberg-table-format') %} {%- endif %} {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%} From 8e027d8fcfbf3958cbc97dfe38fba153d1077a6d Mon Sep 17 00:00:00 2001 From: pei Date: Wed, 20 Nov 2024 10:31:44 +0900 Subject: [PATCH 31/41] Handle non-English Snowflake error message for non-existing schemas (#840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Handle Japanese Snowflake error message for non-existing schemas * impl.py を更新 Co-authored-by: Anders * Changelog entry * Update dbt/adapters/snowflake/impl.py Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --------- Co-authored-by: Anders Co-authored-by: Doug Beatty Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Fixes-20231129-124145.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231129-124145.yaml diff --git a/.changes/unreleased/Fixes-20231129-124145.yaml b/.changes/unreleased/Fixes-20231129-124145.yaml new file mode 100644 index 000000000..72a889d17 --- /dev/null +++ b/.changes/unreleased/Fixes-20231129-124145.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Handle non-English Snowflake error message for non-existing schemas +time: 2023-11-29T12:41:45.1273-07:00 +custom: + Author: pei0804 + Issue: "834" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index dc256c1cb..ac0d903db 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -257,7 +257,9 @@ def list_relations_without_caching( # if the schema doesn't exist, we just want to return. # Alternatively, we could query the list of schemas before we start # and skip listing the missing ones, which sounds expensive. - if "Object does not exist" in str(exc): + # "002043 (02000)" is error code for "object does not exist or is not found" + # The error message text may vary across languages, but the error code is expected to be more stable + if "002043 (02000)" in str(exc): return [] raise From 54999b260f54b5a1e0e52a0e6d0b4a683b064aae Mon Sep 17 00:00:00 2001 From: FishtownBuildBot <77737458+FishtownBuildBot@users.noreply.github.com> Date: Sun, 1 Dec 2024 21:53:45 -0500 Subject: [PATCH 32/41] Cleanup main after cutting new 1.9.latest branch (#1255) * Clean up changelog on main * Bumping version to 1.10.0a1 * Code quality cleanup --- .bumpversion.cfg | 2 +- .changes/1.9.0-b1.md | 61 ------------------ .../1.9.0/Dependencies-20231219-125152.yaml | 6 -- .../1.9.0/Dependencies-20240412-155921.yaml | 6 -- .../1.9.0/Dependencies-20240429-124038.yaml | 6 -- .../1.9.0/Dependencies-20240429-124044.yaml | 6 -- .../1.9.0/Dependencies-20240624-122538.yaml | 6 -- .../1.9.0/Dependencies-20240718-120848.yaml | 6 -- .../1.9.0/Dependencies-20240718-120849.yaml | 6 -- .../1.9.0/Dependencies-20240718-120852.yaml | 6 -- .../1.9.0/Dependencies-20240718-120857.yaml | 6 -- .../1.9.0/Dependencies-20240719-120828.yaml | 6 -- .changes/1.9.0/Features-20240131-125318.yaml | 6 -- .changes/1.9.0/Features-20240430-185714.yaml | 6 -- .changes/1.9.0/Features-20240501-151901.yaml | 6 -- .changes/1.9.0/Features-20240604-154856.yaml | 6 -- .changes/1.9.0/Features-20240610-171026.yaml | 6 -- .changes/1.9.0/Features-20240709-194316.yaml | 6 -- .changes/1.9.0/Features-20240710-172345.yaml | 6 -- .changes/1.9.0/Features-20240911-001806.yaml | 6 -- .changes/1.9.0/Features-20240913-215416.yaml | 6 -- .changes/1.9.0/Features-20240917-100505.yaml | 6 -- .changes/1.9.0/Features-20240923-203204.yaml | 6 -- .changes/1.9.0/Features-20240930-112041.yaml | 6 -- .changes/1.9.0/Fixes-20240516-174337.yaml | 6 -- .changes/1.9.0/Fixes-20240516-224134.yaml | 6 -- .changes/1.9.0/Fixes-20240522-160538.yaml | 6 -- .changes/1.9.0/Fixes-20240605-125611.yaml | 6 -- .changes/1.9.0/Fixes-20240607-102708.yaml | 6 -- .changes/1.9.0/Fixes-20240628-190140.yaml | 7 --- .changes/1.9.0/Fixes-20240705-165932.yaml | 6 -- .changes/1.9.0/Fixes-20240920-193613.yaml | 6 -- .../1.9.0/Under the Hood-20240327-001304.yaml | 6 -- .../1.9.0/Under the Hood-20240425-144556.yaml | 6 -- .../1.9.0/Under the Hood-20240517-143743.yaml | 6 -- .../1.9.0/Under the Hood-20240614-170858.yaml | 6 -- .../1.9.0/Under the Hood-20240716-174655.yaml | 6 -- .../1.9.0/Under the Hood-20240719-125618.yaml | 6 -- .../1.9.0/Under the Hood-20240722-143114.yaml | 6 -- .../1.9.0/Under the Hood-20240806-215935.yaml | 6 -- .../1.9.0/Under the Hood-20240917-181147.yaml | 6 -- .../Breaking Changes-20241016-183143.yaml | 6 -- .../unreleased/Features-20241107-170307.yaml | 7 --- .../unreleased/Fixes-20231129-124145.yaml | 6 -- .../unreleased/Fixes-20241008-122635.yaml | 6 -- .../unreleased/Fixes-20241104-104610.yaml | 7 --- .../unreleased/Fixes-20241104-172349.yaml | 6 -- .../Under the Hood-20241016-035544.yaml | 6 -- .../Under the Hood-20241106-113249.yaml | 6 -- .../Under the Hood-20241118-231637.yaml | 6 -- CHANGELOG.md | 63 ------------------- dbt/adapters/snowflake/__version__.py | 2 +- 52 files changed, 2 insertions(+), 417 deletions(-) delete mode 100644 .changes/1.9.0-b1.md delete mode 100644 .changes/1.9.0/Dependencies-20231219-125152.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240412-155921.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240429-124038.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240429-124044.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240624-122538.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120848.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120849.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120852.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240718-120857.yaml delete mode 100644 .changes/1.9.0/Dependencies-20240719-120828.yaml delete mode 100644 .changes/1.9.0/Features-20240131-125318.yaml delete mode 100644 .changes/1.9.0/Features-20240430-185714.yaml delete mode 100644 .changes/1.9.0/Features-20240501-151901.yaml delete mode 100644 .changes/1.9.0/Features-20240604-154856.yaml delete mode 100644 .changes/1.9.0/Features-20240610-171026.yaml delete mode 100644 .changes/1.9.0/Features-20240709-194316.yaml delete mode 100644 .changes/1.9.0/Features-20240710-172345.yaml delete mode 100644 .changes/1.9.0/Features-20240911-001806.yaml delete mode 100644 .changes/1.9.0/Features-20240913-215416.yaml delete mode 100644 .changes/1.9.0/Features-20240917-100505.yaml delete mode 100644 .changes/1.9.0/Features-20240923-203204.yaml delete mode 100644 .changes/1.9.0/Features-20240930-112041.yaml delete mode 100644 .changes/1.9.0/Fixes-20240516-174337.yaml delete mode 100644 .changes/1.9.0/Fixes-20240516-224134.yaml delete mode 100644 .changes/1.9.0/Fixes-20240522-160538.yaml delete mode 100644 .changes/1.9.0/Fixes-20240605-125611.yaml delete mode 100644 .changes/1.9.0/Fixes-20240607-102708.yaml delete mode 100644 .changes/1.9.0/Fixes-20240628-190140.yaml delete mode 100644 .changes/1.9.0/Fixes-20240705-165932.yaml delete mode 100644 .changes/1.9.0/Fixes-20240920-193613.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240327-001304.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240425-144556.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240517-143743.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240614-170858.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240716-174655.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240719-125618.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240722-143114.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240806-215935.yaml delete mode 100644 .changes/1.9.0/Under the Hood-20240917-181147.yaml delete mode 100644 .changes/unreleased/Breaking Changes-20241016-183143.yaml delete mode 100644 .changes/unreleased/Features-20241107-170307.yaml delete mode 100644 .changes/unreleased/Fixes-20231129-124145.yaml delete mode 100644 .changes/unreleased/Fixes-20241008-122635.yaml delete mode 100644 .changes/unreleased/Fixes-20241104-104610.yaml delete mode 100644 .changes/unreleased/Fixes-20241104-172349.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241016-035544.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241106-113249.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241118-231637.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7ff98322d..1d774733d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0b1 +current_version = 1.10.0a1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md deleted file mode 100644 index 15a01afe7..000000000 --- a/.changes/1.9.0-b1.md +++ /dev/null @@ -1,61 +0,0 @@ -## dbt-snowflake 1.9.0-b1 - October 01, 2024 - -### Features - -- Support refresh_mode and initialize parameters for dynamic tables ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- Add tests for cross-database `cast` macro ([#1009](https://github.com/dbt-labs/dbt-snowflake/issues/1009)) -- Cross-database `date` macro ([#1013](https://github.com/dbt-labs/dbt-snowflake/issues/1013)) -- Replace underscores with hyphens in account IDs to prevent SSL issues ([#1068](https://github.com/dbt-labs/dbt-snowflake/issues/1068)) -- Support JWT Authentication ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- Improve run times for large projects by reusing connections by default ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Improve run times when using key pair auth by caching the private key ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Add support for Iceberg table materializations. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Microbatch incremental strategy ([#1182](https://github.com/dbt-labs/dbt-snowflake/issues/1182)) -- Add support for Iceberg table format in Dynamic Tables ([#1183](https://github.com/dbt-labs/dbt-snowflake/issues/1183)) -- Add Iceberg format Incremental Models ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Add support for all on_schema_change incremental model strategies. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Fixes - -- Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability ([#1048](https://github.com/dbt-labs/dbt-snowflake/issues/1048)) -- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) -- Rename targets for tables and views use fully qualified names ([#1031](https://github.com/dbt-labs/dbt-snowflake/issues/1031)) -- Surface SSO token expiration in logs ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) -- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) -- Use show ... starts with instead of show ... like in _show_object_metadata ([#1102](https://github.com/dbt-labs/dbt-snowflake/issues/1102)) -- Fix issue where dbt-snowflake attempts to drop database roles during grants sync ([#1151](https://github.com/dbt-labs/dbt-snowflake/issues/1151)) - -### Under the Hood - -- Lazy load agate ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- Speedup catalog string comparison by using ilike before equals ([#1035](https://github.com/dbt-labs/dbt-snowflake/issues/1035)) -- Improve memory efficiency of the process_results() override. ([#1053](https://github.com/dbt-labs/dbt-snowflake/issues/1053)) -- Automate all manual integration tests for Dynamic Tables ([#1084](https://github.com/dbt-labs/dbt-snowflake/issues/1084)) -- Add support for experimental record/replay testing. ([#1106](https://github.com/dbt-labs/dbt-snowflake/issues/1106)) -- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-snowflake/issues/1136)) -- Add support for Python 3.12 ([#903](https://github.com/dbt-labs/dbt-snowflake/issues/903)) -- Isolating distribution testing ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- Change behavior flag semantics to log iceberg flag warnings.. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Dependencies - -- Update freezegun requirement from ~=1.3 to ~=1.4 ([#869](https://github.com/dbt-labs/dbt-snowflake/pull/869)) -- Bump actions/upload-artifact from 3 to 4 ([#971](https://github.com/dbt-labs/dbt-snowflake/pull/971)) -- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1006](https://github.com/dbt-labs/dbt-snowflake/pull/1006)) -- Bump actions/download-artifact from 3 to 4 ([#1007](https://github.com/dbt-labs/dbt-snowflake/pull/1007)) -- Bump aurelien-baudet/workflow-dispatch from 2 to 4 ([#1093](https://github.com/dbt-labs/dbt-snowflake/pull/1093)) -- Update twine requirement from ~=4.0 to ~=5.1 ([#1120](https://github.com/dbt-labs/dbt-snowflake/pull/1120)) -- Bump pre-commit from 3.7.0 to 3.7.1 ([#1119](https://github.com/dbt-labs/dbt-snowflake/pull/1119)) -- Update wheel requirement from ~=0.42 to ~=0.43 ([#1121](https://github.com/dbt-labs/dbt-snowflake/pull/1121)) -- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1122](https://github.com/dbt-labs/dbt-snowflake/pull/1122)) -- Update tox requirement from ~=4.11 to ~=4.16 ([#1135](https://github.com/dbt-labs/dbt-snowflake/pull/1135)) - -### Contributors -- [@HenkvanDyk,mikealfare](https://github.com/HenkvanDyk,mikealfare) ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- [@McKnight-42](https://github.com/McKnight-42) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851), [#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- [@amardatar](https://github.com/amardatar) ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- [@dwreeves](https://github.com/dwreeves) ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- [@leahwicz](https://github.com/leahwicz) ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- [@llam15](https://github.com/llam15) ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- [@mikealfare,](https://github.com/mikealfare,) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) diff --git a/.changes/1.9.0/Dependencies-20231219-125152.yaml b/.changes/1.9.0/Dependencies-20231219-125152.yaml deleted file mode 100644 index 2d730daf1..000000000 --- a/.changes/1.9.0/Dependencies-20231219-125152.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update freezegun requirement from ~=1.3 to ~=1.4" -time: 2023-12-19T12:51:52.00000Z -custom: - Author: dependabot[bot] - PR: 869 diff --git a/.changes/1.9.0/Dependencies-20240412-155921.yaml b/.changes/1.9.0/Dependencies-20240412-155921.yaml deleted file mode 100644 index f83e5b404..000000000 --- a/.changes/1.9.0/Dependencies-20240412-155921.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump actions/upload-artifact from 3 to 4" -time: 2024-04-12T15:59:21.00000Z -custom: - Author: dependabot[bot] - PR: 971 diff --git a/.changes/1.9.0/Dependencies-20240429-124038.yaml b/.changes/1.9.0/Dependencies-20240429-124038.yaml deleted file mode 100644 index 5fa954c8a..000000000 --- a/.changes/1.9.0/Dependencies-20240429-124038.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump dbt-labs/actions from 1.1.0 to 1.1.1" -time: 2024-04-29T12:40:38.00000Z -custom: - Author: dependabot[bot] - PR: 1006 diff --git a/.changes/1.9.0/Dependencies-20240429-124044.yaml b/.changes/1.9.0/Dependencies-20240429-124044.yaml deleted file mode 100644 index 834fce096..000000000 --- a/.changes/1.9.0/Dependencies-20240429-124044.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump actions/download-artifact from 3 to 4" -time: 2024-04-29T12:40:44.00000Z -custom: - Author: dependabot[bot] - PR: 1007 diff --git a/.changes/1.9.0/Dependencies-20240624-122538.yaml b/.changes/1.9.0/Dependencies-20240624-122538.yaml deleted file mode 100644 index e47731aef..000000000 --- a/.changes/1.9.0/Dependencies-20240624-122538.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump aurelien-baudet/workflow-dispatch from 2 to 4" -time: 2024-06-24T12:25:38.00000Z -custom: - Author: dependabot[bot] - PR: 1093 diff --git a/.changes/1.9.0/Dependencies-20240718-120848.yaml b/.changes/1.9.0/Dependencies-20240718-120848.yaml deleted file mode 100644 index c46a30eba..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120848.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update twine requirement from ~=4.0 to ~=5.1" -time: 2024-07-18T12:08:48.00000Z -custom: - Author: dependabot[bot] - PR: 1120 diff --git a/.changes/1.9.0/Dependencies-20240718-120849.yaml b/.changes/1.9.0/Dependencies-20240718-120849.yaml deleted file mode 100644 index df248ff7d..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120849.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump pre-commit from 3.7.0 to 3.7.1" -time: 2024-07-18T12:08:49.00000Z -custom: - Author: dependabot[bot] - PR: 1119 diff --git a/.changes/1.9.0/Dependencies-20240718-120852.yaml b/.changes/1.9.0/Dependencies-20240718-120852.yaml deleted file mode 100644 index 40c171f93..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120852.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update wheel requirement from ~=0.42 to ~=0.43" -time: 2024-07-18T12:08:52.00000Z -custom: - Author: dependabot[bot] - PR: 1121 diff --git a/.changes/1.9.0/Dependencies-20240718-120857.yaml b/.changes/1.9.0/Dependencies-20240718-120857.yaml deleted file mode 100644 index e4bfe04d0..000000000 --- a/.changes/1.9.0/Dependencies-20240718-120857.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.5 to ~=3.6" -time: 2024-07-18T12:08:57.00000Z -custom: - Author: dependabot[bot] - PR: 1122 diff --git a/.changes/1.9.0/Dependencies-20240719-120828.yaml b/.changes/1.9.0/Dependencies-20240719-120828.yaml deleted file mode 100644 index ea7af843c..000000000 --- a/.changes/1.9.0/Dependencies-20240719-120828.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.11 to ~=4.16" -time: 2024-07-19T12:08:28.00000Z -custom: - Author: dependabot[bot] - PR: 1135 diff --git a/.changes/1.9.0/Features-20240131-125318.yaml b/.changes/1.9.0/Features-20240131-125318.yaml deleted file mode 100644 index 63771d71e..000000000 --- a/.changes/1.9.0/Features-20240131-125318.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support refresh_mode and initialize parameters for dynamic tables -time: 2024-01-31T12:53:18.111616Z -custom: - Author: HenkvanDyk,mikealfare - Issue: "1076" diff --git a/.changes/1.9.0/Features-20240430-185714.yaml b/.changes/1.9.0/Features-20240430-185714.yaml deleted file mode 100644 index 9fd1e97ea..000000000 --- a/.changes/1.9.0/Features-20240430-185714.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add tests for cross-database `cast` macro -time: 2024-04-30T18:57:14.753057-06:00 -custom: - Author: dbeatty10 - Issue: "1009" diff --git a/.changes/1.9.0/Features-20240501-151901.yaml b/.changes/1.9.0/Features-20240501-151901.yaml deleted file mode 100644 index 0f792c40e..000000000 --- a/.changes/1.9.0/Features-20240501-151901.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Cross-database `date` macro -time: 2024-05-01T15:19:01.141157-06:00 -custom: - Author: dbeatty10 - Issue: 1013 diff --git a/.changes/1.9.0/Features-20240604-154856.yaml b/.changes/1.9.0/Features-20240604-154856.yaml deleted file mode 100644 index 7d83b1da7..000000000 --- a/.changes/1.9.0/Features-20240604-154856.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Replace underscores with hyphens in account IDs to prevent SSL issues -time: 2024-06-04T15:48:56.845374-07:00 -custom: - Author: colin-rogers-dbt - Issue: "1068" diff --git a/.changes/1.9.0/Features-20240610-171026.yaml b/.changes/1.9.0/Features-20240610-171026.yaml deleted file mode 100644 index 5cc055160..000000000 --- a/.changes/1.9.0/Features-20240610-171026.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support JWT Authentication -time: 2024-06-10T17:10:26.421463-04:00 -custom: - Author: llam15 - Issue: 1079 726 diff --git a/.changes/1.9.0/Features-20240709-194316.yaml b/.changes/1.9.0/Features-20240709-194316.yaml deleted file mode 100644 index a867387e3..000000000 --- a/.changes/1.9.0/Features-20240709-194316.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Improve run times for large projects by reusing connections by default -time: 2024-07-09T19:43:16.489649-04:00 -custom: - Author: mikealfare amardatar - Issue: "1082" diff --git a/.changes/1.9.0/Features-20240710-172345.yaml b/.changes/1.9.0/Features-20240710-172345.yaml deleted file mode 100644 index e68f63812..000000000 --- a/.changes/1.9.0/Features-20240710-172345.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Improve run times when using key pair auth by caching the private key -time: 2024-07-10T17:23:45.046905-04:00 -custom: - Author: mikealfare aranke - Issue: "1082" diff --git a/.changes/1.9.0/Features-20240911-001806.yaml b/.changes/1.9.0/Features-20240911-001806.yaml deleted file mode 100644 index 024480b96..000000000 --- a/.changes/1.9.0/Features-20240911-001806.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for Iceberg table materializations. -time: 2024-09-11T00:18:06.780586-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/1.9.0/Features-20240913-215416.yaml b/.changes/1.9.0/Features-20240913-215416.yaml deleted file mode 100644 index b2a6e556e..000000000 --- a/.changes/1.9.0/Features-20240913-215416.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Microbatch incremental strategy -time: 2024-09-13T21:54:16.492885-04:00 -custom: - Author: michelleark - Issue: "1182" diff --git a/.changes/1.9.0/Features-20240917-100505.yaml b/.changes/1.9.0/Features-20240917-100505.yaml deleted file mode 100644 index 22cabc904..000000000 --- a/.changes/1.9.0/Features-20240917-100505.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for Iceberg table format in Dynamic Tables -time: 2024-09-17T10:05:05.609859-04:00 -custom: - Author: mikealfare - Issue: "1183" diff --git a/.changes/1.9.0/Features-20240923-203204.yaml b/.changes/1.9.0/Features-20240923-203204.yaml deleted file mode 100644 index eaca4906b..000000000 --- a/.changes/1.9.0/Features-20240923-203204.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add Iceberg format Incremental Models -time: 2024-09-23T20:32:04.783741-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/1.9.0/Features-20240930-112041.yaml b/.changes/1.9.0/Features-20240930-112041.yaml deleted file mode 100644 index 1395a8bf7..000000000 --- a/.changes/1.9.0/Features-20240930-112041.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add support for all on_schema_change incremental model strategies. -time: 2024-09-30T11:20:41.99589-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/1.9.0/Fixes-20240516-174337.yaml b/.changes/1.9.0/Fixes-20240516-174337.yaml deleted file mode 100644 index 955d90ed3..000000000 --- a/.changes/1.9.0/Fixes-20240516-174337.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle -time: 2024-05-16T17:43:37.336858-04:00 -custom: - Author: mikealfare - Issue: "1016" diff --git a/.changes/1.9.0/Fixes-20240516-224134.yaml b/.changes/1.9.0/Fixes-20240516-224134.yaml deleted file mode 100644 index 011ecb449..000000000 --- a/.changes/1.9.0/Fixes-20240516-224134.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability -time: 2024-05-16T22:41:34.256095+01:00 -custom: - Author: aranke - Issue: "1048" diff --git a/.changes/1.9.0/Fixes-20240522-160538.yaml b/.changes/1.9.0/Fixes-20240522-160538.yaml deleted file mode 100644 index 4921706a9..000000000 --- a/.changes/1.9.0/Fixes-20240522-160538.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: 'Rename targets for tables and views use fully qualified names' -time: 2024-05-22T16:05:38.602074-04:00 -custom: - Author: mikealfare - Issue: "1031" diff --git a/.changes/1.9.0/Fixes-20240605-125611.yaml b/.changes/1.9.0/Fixes-20240605-125611.yaml deleted file mode 100644 index c4560774c..000000000 --- a/.changes/1.9.0/Fixes-20240605-125611.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Surface SSO token expiration in logs -time: 2024-06-05T12:56:11.802237-04:00 -custom: - Author: mikealfare, McKnight-42 - Issue: "851" diff --git a/.changes/1.9.0/Fixes-20240607-102708.yaml b/.changes/1.9.0/Fixes-20240607-102708.yaml deleted file mode 100644 index 58cd9bbee..000000000 --- a/.changes/1.9.0/Fixes-20240607-102708.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: return to previous naming convention to return to quoting policy -time: 2024-06-07T10:27:08.542159-05:00 -custom: - Author: McKnight-42 - Issue: "1074" diff --git a/.changes/1.9.0/Fixes-20240628-190140.yaml b/.changes/1.9.0/Fixes-20240628-190140.yaml deleted file mode 100644 index c58b465fd..000000000 --- a/.changes/1.9.0/Fixes-20240628-190140.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Fix scenario where using the `--empty` flag causes metadata queries to contain - limit clauses -time: 2024-06-28T19:01:40.558234-04:00 -custom: - Author: mikealfare - Issue: "1033" diff --git a/.changes/1.9.0/Fixes-20240705-165932.yaml b/.changes/1.9.0/Fixes-20240705-165932.yaml deleted file mode 100644 index ffe902c92..000000000 --- a/.changes/1.9.0/Fixes-20240705-165932.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Use show ... starts with instead of show ... like in _show_object_metadata -time: 2024-07-05T16:59:32.087555+01:00 -custom: - Author: aranke - Issue: "1102" diff --git a/.changes/1.9.0/Fixes-20240920-193613.yaml b/.changes/1.9.0/Fixes-20240920-193613.yaml deleted file mode 100644 index f85f6fc56..000000000 --- a/.changes/1.9.0/Fixes-20240920-193613.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix issue where dbt-snowflake attempts to drop database roles during grants sync -time: 2024-09-20T19:36:13.671173-04:00 -custom: - Author: mikealfare - Issue: "1151" diff --git a/.changes/1.9.0/Under the Hood-20240327-001304.yaml b/.changes/1.9.0/Under the Hood-20240327-001304.yaml deleted file mode 100644 index 3e823ec86..000000000 --- a/.changes/1.9.0/Under the Hood-20240327-001304.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Lazy load agate -time: 2024-03-27T00:13:04.246062-04:00 -custom: - Author: dwreeves - Issue: "953" diff --git a/.changes/1.9.0/Under the Hood-20240425-144556.yaml b/.changes/1.9.0/Under the Hood-20240425-144556.yaml deleted file mode 100644 index 002da3c1f..000000000 --- a/.changes/1.9.0/Under the Hood-20240425-144556.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Speedup catalog string comparison by using ilike before equals -time: 2024-04-25T14:45:56.549787+02:00 -custom: - Author: aranke - Issue: '1035' diff --git a/.changes/1.9.0/Under the Hood-20240517-143743.yaml b/.changes/1.9.0/Under the Hood-20240517-143743.yaml deleted file mode 100644 index 598c60ad4..000000000 --- a/.changes/1.9.0/Under the Hood-20240517-143743.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Improve memory efficiency of the process_results() override. -time: 2024-05-17T14:37:43.7414-04:00 -custom: - Author: peterallenwebb - Issue: "1053" diff --git a/.changes/1.9.0/Under the Hood-20240614-170858.yaml b/.changes/1.9.0/Under the Hood-20240614-170858.yaml deleted file mode 100644 index cc806726b..000000000 --- a/.changes/1.9.0/Under the Hood-20240614-170858.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Automate all manual integration tests for Dynamic Tables -time: 2024-06-14T17:08:58.231472-04:00 -custom: - Author: mikealfare - Issue: "1084" diff --git a/.changes/1.9.0/Under the Hood-20240716-174655.yaml b/.changes/1.9.0/Under the Hood-20240716-174655.yaml deleted file mode 100644 index 14c3c8d76..000000000 --- a/.changes/1.9.0/Under the Hood-20240716-174655.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add support for experimental record/replay testing. -time: 2024-07-16T17:46:55.11204-04:00 -custom: - Author: peterallenwebb - Issue: "1106" diff --git a/.changes/1.9.0/Under the Hood-20240719-125618.yaml b/.changes/1.9.0/Under the Hood-20240719-125618.yaml deleted file mode 100644 index 3d90b732c..000000000 --- a/.changes/1.9.0/Under the Hood-20240719-125618.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove `freezegun` as a testing dependency; this package is no longer used -time: 2024-07-19T12:56:18.957049-04:00 -custom: - Author: mikealfare - Issue: "1136" diff --git a/.changes/1.9.0/Under the Hood-20240722-143114.yaml b/.changes/1.9.0/Under the Hood-20240722-143114.yaml deleted file mode 100644 index dc5c2dbb1..000000000 --- a/.changes/1.9.0/Under the Hood-20240722-143114.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add support for Python 3.12 -time: 2024-07-22T14:31:14.024865-07:00 -custom: - Author: versusfacit - Issue: "903" diff --git a/.changes/1.9.0/Under the Hood-20240806-215935.yaml b/.changes/1.9.0/Under the Hood-20240806-215935.yaml deleted file mode 100644 index 660918350..000000000 --- a/.changes/1.9.0/Under the Hood-20240806-215935.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Isolating distribution testing -time: 2024-08-06T21:59:35.284641-04:00 -custom: - Author: leahwicz - Issue: "1130" diff --git a/.changes/1.9.0/Under the Hood-20240917-181147.yaml b/.changes/1.9.0/Under the Hood-20240917-181147.yaml deleted file mode 100644 index 2f52174dd..000000000 --- a/.changes/1.9.0/Under the Hood-20240917-181147.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Change behavior flag semantics to log iceberg flag warnings.. -time: 2024-09-17T18:11:47.525026-07:00 -custom: - Author: versusfacit - Issue: "321" diff --git a/.changes/unreleased/Breaking Changes-20241016-183143.yaml b/.changes/unreleased/Breaking Changes-20241016-183143.yaml deleted file mode 100644 index 26cc4b6de..000000000 --- a/.changes/unreleased/Breaking Changes-20241016-183143.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Breaking Changes -body: Drop support for Python 3.8 -time: 2024-10-16T18:31:43.4167-04:00 -custom: - Author: mikealfare - Issue: "1211" diff --git a/.changes/unreleased/Features-20241107-170307.yaml b/.changes/unreleased/Features-20241107-170307.yaml deleted file mode 100644 index 1479c5805..000000000 --- a/.changes/unreleased/Features-20241107-170307.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: 'Allow configurable pagination on list_relations_without_caching to support - users with a large number of objects per schema' -time: 2024-11-07T17:03:07.826352-05:00 -custom: - Author: mikealfare - Issue: "1234" diff --git a/.changes/unreleased/Fixes-20231129-124145.yaml b/.changes/unreleased/Fixes-20231129-124145.yaml deleted file mode 100644 index 72a889d17..000000000 --- a/.changes/unreleased/Fixes-20231129-124145.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Handle non-English Snowflake error message for non-existing schemas -time: 2023-11-29T12:41:45.1273-07:00 -custom: - Author: pei0804 - Issue: "834" diff --git a/.changes/unreleased/Fixes-20241008-122635.yaml b/.changes/unreleased/Fixes-20241008-122635.yaml deleted file mode 100644 index c069283d6..000000000 --- a/.changes/unreleased/Fixes-20241008-122635.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Dynamic Iceberg table base_location_subpath generation fix. -time: 2024-10-08T12:26:35.521308-07:00 -custom: - Author: versusfacit - Issue: "1200" diff --git a/.changes/unreleased/Fixes-20241104-104610.yaml b/.changes/unreleased/Fixes-20241104-104610.yaml deleted file mode 100644 index c512d0bdd..000000000 --- a/.changes/unreleased/Fixes-20241104-104610.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: 'Performance fixes for snowflake microbatch strategy: use temp view instead - of table, remove unnecessary ''using'' clause' -time: 2024-11-04T10:46:10.005317-05:00 -custom: - Author: michelleark - Issue: "1228" diff --git a/.changes/unreleased/Fixes-20241104-172349.yaml b/.changes/unreleased/Fixes-20241104-172349.yaml deleted file mode 100644 index 07c90d93c..000000000 --- a/.changes/unreleased/Fixes-20241104-172349.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Iceberg quoting ignore fix. -time: 2024-11-04T17:23:49.706297-08:00 -custom: - Author: versusfacit - Issue: "1227" diff --git a/.changes/unreleased/Under the Hood-20241016-035544.yaml b/.changes/unreleased/Under the Hood-20241016-035544.yaml deleted file mode 100644 index 59e4f70de..000000000 --- a/.changes/unreleased/Under the Hood-20241016-035544.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add telemetry function -time: 2024-10-16T03:55:44.144174-07:00 -custom: - Author: versusfacit - Issue: "301" diff --git a/.changes/unreleased/Under the Hood-20241106-113249.yaml b/.changes/unreleased/Under the Hood-20241106-113249.yaml deleted file mode 100644 index 0437a8c88..000000000 --- a/.changes/unreleased/Under the Hood-20241106-113249.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: remove SnowflakeAdapterResponse in favor of updated AdapterResponse in base -time: 2024-11-06T11:32:49.503467-08:00 -custom: - Author: colin-rogers-dbt - Issue: "1233" diff --git a/.changes/unreleased/Under the Hood-20241118-231637.yaml b/.changes/unreleased/Under the Hood-20241118-231637.yaml deleted file mode 100644 index f787a2a79..000000000 --- a/.changes/unreleased/Under the Hood-20241118-231637.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add iceberg docs url to behavior flag. -time: 2024-11-18T23:16:37.926576-08:00 -custom: - Author: versusfacit - Issue: "210" diff --git a/CHANGELOG.md b/CHANGELOG.md index 599c20195..8b6702b8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,69 +5,6 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-snowflake/blob/main/CONTRIBUTING.md#adding-changelog-entry) -## dbt-snowflake 1.9.0-b1 - October 01, 2024 - -### Features - -- Support refresh_mode and initialize parameters for dynamic tables ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- Add tests for cross-database `cast` macro ([#1009](https://github.com/dbt-labs/dbt-snowflake/issues/1009)) -- Cross-database `date` macro ([#1013](https://github.com/dbt-labs/dbt-snowflake/issues/1013)) -- Replace underscores with hyphens in account IDs to prevent SSL issues ([#1068](https://github.com/dbt-labs/dbt-snowflake/issues/1068)) -- Support JWT Authentication ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- Improve run times for large projects by reusing connections by default ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Improve run times when using key pair auth by caching the private key ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- Add support for Iceberg table materializations. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Microbatch incremental strategy ([#1182](https://github.com/dbt-labs/dbt-snowflake/issues/1182)) -- Add support for Iceberg table format in Dynamic Tables ([#1183](https://github.com/dbt-labs/dbt-snowflake/issues/1183)) -- Add Iceberg format Incremental Models ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) -- Add support for all on_schema_change incremental model strategies. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Fixes - -- Get catalog metadata for a single relation in the most optimized way using the get_catalog_for_single_relation macro and capability ([#1048](https://github.com/dbt-labs/dbt-snowflake/issues/1048)) -- Update relation caching to correctly identify dynamic tables, accounting for Snowflake's `2024_03` bundle ([#1016](https://github.com/dbt-labs/dbt-snowflake/issues/1016)) -- Rename targets for tables and views use fully qualified names ([#1031](https://github.com/dbt-labs/dbt-snowflake/issues/1031)) -- Surface SSO token expiration in logs ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) -- return to previous naming convention to return to quoting policy ([#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#1033](https://github.com/dbt-labs/dbt-snowflake/issues/1033)) -- Use show ... starts with instead of show ... like in _show_object_metadata ([#1102](https://github.com/dbt-labs/dbt-snowflake/issues/1102)) -- Fix issue where dbt-snowflake attempts to drop database roles during grants sync ([#1151](https://github.com/dbt-labs/dbt-snowflake/issues/1151)) - -### Under the Hood - -- Lazy load agate ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- Speedup catalog string comparison by using ilike before equals ([#1035](https://github.com/dbt-labs/dbt-snowflake/issues/1035)) -- Improve memory efficiency of the process_results() override. ([#1053](https://github.com/dbt-labs/dbt-snowflake/issues/1053)) -- Automate all manual integration tests for Dynamic Tables ([#1084](https://github.com/dbt-labs/dbt-snowflake/issues/1084)) -- Add support for experimental record/replay testing. ([#1106](https://github.com/dbt-labs/dbt-snowflake/issues/1106)) -- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-snowflake/issues/1136)) -- Add support for Python 3.12 ([#903](https://github.com/dbt-labs/dbt-snowflake/issues/903)) -- Isolating distribution testing ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- Change behavior flag semantics to log iceberg flag warnings.. ([#321](https://github.com/dbt-labs/dbt-snowflake/issues/321)) - -### Dependencies - -- Update freezegun requirement from ~=1.3 to ~=1.4 ([#869](https://github.com/dbt-labs/dbt-snowflake/pull/869)) -- Bump actions/upload-artifact from 3 to 4 ([#971](https://github.com/dbt-labs/dbt-snowflake/pull/971)) -- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1006](https://github.com/dbt-labs/dbt-snowflake/pull/1006)) -- Bump actions/download-artifact from 3 to 4 ([#1007](https://github.com/dbt-labs/dbt-snowflake/pull/1007)) -- Bump aurelien-baudet/workflow-dispatch from 2 to 4 ([#1093](https://github.com/dbt-labs/dbt-snowflake/pull/1093)) -- Update twine requirement from ~=4.0 to ~=5.1 ([#1120](https://github.com/dbt-labs/dbt-snowflake/pull/1120)) -- Bump pre-commit from 3.7.0 to 3.7.1 ([#1119](https://github.com/dbt-labs/dbt-snowflake/pull/1119)) -- Update wheel requirement from ~=0.42 to ~=0.43 ([#1121](https://github.com/dbt-labs/dbt-snowflake/pull/1121)) -- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1122](https://github.com/dbt-labs/dbt-snowflake/pull/1122)) -- Update tox requirement from ~=4.11 to ~=4.16 ([#1135](https://github.com/dbt-labs/dbt-snowflake/pull/1135)) - -### Contributors -- [@HenkvanDyk,mikealfare](https://github.com/HenkvanDyk,mikealfare) ([#1076](https://github.com/dbt-labs/dbt-snowflake/issues/1076)) -- [@McKnight-42](https://github.com/McKnight-42) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851), [#1074](https://github.com/dbt-labs/dbt-snowflake/issues/1074)) -- [@amardatar](https://github.com/amardatar) ([#1082](https://github.com/dbt-labs/dbt-snowflake/issues/1082)) -- [@dwreeves](https://github.com/dwreeves) ([#953](https://github.com/dbt-labs/dbt-snowflake/issues/953)) -- [@leahwicz](https://github.com/leahwicz) ([#1130](https://github.com/dbt-labs/dbt-snowflake/issues/1130)) -- [@llam15](https://github.com/llam15) ([#1079](https://github.com/dbt-labs/dbt-snowflake/issues/1079), [#726](https://github.com/dbt-labs/dbt-snowflake/issues/726)) -- [@mikealfare,](https://github.com/mikealfare,) ([#851](https://github.com/dbt-labs/dbt-snowflake/issues/851)) - - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-snowflake/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/snowflake/__version__.py b/dbt/adapters/snowflake/__version__.py index a4077fff2..1af777a62 100644 --- a/dbt/adapters/snowflake/__version__.py +++ b/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = "1.9.0b1" +version = "1.10.0a1" From 5cc5d224f9cc45802b8141472a594c2cdf0facb1 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 2 Dec 2024 11:05:29 -0500 Subject: [PATCH 33/41] Use timestamp_tz type in microbatch `delete` DDL (#1257) --- .changes/unreleased/Fixes-20241127-162204.yaml | 6 ++++++ .../snowflake/macros/materializations/merge.sql | 4 ++-- .../adapter/test_incremental_microbatch.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241127-162204.yaml diff --git a/.changes/unreleased/Fixes-20241127-162204.yaml b/.changes/unreleased/Fixes-20241127-162204.yaml new file mode 100644 index 000000000..2b990b1f9 --- /dev/null +++ b/.changes/unreleased/Fixes-20241127-162204.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use timestamp_tz type in microbatch `delete` DDL +time: 2024-11-27T16:22:04.103212-05:00 +custom: + Author: michelleark + Issue: "1256" diff --git a/dbt/include/snowflake/macros/materializations/merge.sql b/dbt/include/snowflake/macros/materializations/merge.sql index c8ac8d6fd..716a325c1 100644 --- a/dbt/include/snowflake/macros/materializations/merge.sql +++ b/dbt/include/snowflake/macros/materializations/merge.sql @@ -58,10 +58,10 @@ {#-- Add additional incremental_predicates to filter for batch --#} {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} - {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= TIMESTAMP '" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "'") %} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "')") %} {% endif %} {% if model.config.__dbt_internal_microbatch_event_time_end -%} - {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < TIMESTAMP '" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "'") %} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "')") %} {% endif %} {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} diff --git a/tests/functional/adapter/test_incremental_microbatch.py b/tests/functional/adapter/test_incremental_microbatch.py index f228c370c..f087596e1 100644 --- a/tests/functional/adapter/test_incremental_microbatch.py +++ b/tests/functional/adapter/test_incremental_microbatch.py @@ -3,6 +3,16 @@ BaseMicrobatch, ) +# Create input with UTC timestamps +_input_model_sql = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, to_timestamp_tz('2020-01-01 00:00:00-0') as event_time +union all +select 2 as id, to_timestamp_tz('2020-01-02 00:00:00-0') as event_time +union all +select 3 as id, to_timestamp_tz('2020-01-03 00:00:00-0') as event_time +""" + # No requirement for a unique_id for snowflake microbatch! _microbatch_model_no_unique_id_sql = """ @@ -16,6 +26,10 @@ class TestSnowflakeMicrobatch(BaseMicrobatch): def microbatch_model_sql(self) -> str: return _microbatch_model_no_unique_id_sql + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + return _input_model_sql + @pytest.fixture(scope="class") def insert_two_rows_sql(self, project) -> str: test_schema_relation = project.adapter.Relation.create( From 0b24a5a2d311ffec5f996ca28532076a637aa6b3 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:05:38 -0800 Subject: [PATCH 34/41] update libpq-dev dependency to 13.18-0+deb11u1 (#1262) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 17315b12d..16060db61 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update \ build-essential=12.9 \ ca-certificates=20210119 \ git=1:2.30.2-1+deb11u2 \ - libpq-dev=13.14-0+deb11u1 \ + libpq-dev=13.18-0+deb11u1 \ make=4.3-4.1 \ openssh-client=1:8.4p1-5+deb11u3 \ software-properties-common=0.96.20.2-2.1 \ From 86cf6e61b904a501d562d56d9fa1caea1c3efd7d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 5 Dec 2024 15:37:57 -0500 Subject: [PATCH 35/41] support MicrobatchConcurrency capability (#1259) --- .changes/unreleased/Features-20241202-095136.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 1 + setup.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20241202-095136.yaml diff --git a/.changes/unreleased/Features-20241202-095136.yaml b/.changes/unreleased/Features-20241202-095136.yaml new file mode 100644 index 000000000..973866a6d --- /dev/null +++ b/.changes/unreleased/Features-20241202-095136.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support MicrobatchConcurrency +time: 2024-12-02T09:51:36.606097-05:00 +custom: + Author: michelleark + Issue: "1260" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index ac0d903db..10ad2a8a1 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -77,6 +77,7 @@ class SnowflakeAdapter(SQLAdapter): Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), Capability.GetCatalogForSingleRelation: CapabilitySupport(support=Support.Full), + Capability.MicrobatchConcurrency: CapabilitySupport(support=Support.Full), } ) diff --git a/setup.py b/setup.py index c0716341d..b237182ce 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters>=1.10.4,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 457c361a8453d7e4c09ce9033f872a651646fd6c Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 5 Dec 2024 16:42:45 -0500 Subject: [PATCH 36/41] [Refactor] use new model.batch context vars (#1258) --- .changes/unreleased/Under the Hood-20241205-144036.yaml | 6 ++++++ dbt/include/snowflake/macros/materializations/merge.sql | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20241205-144036.yaml diff --git a/.changes/unreleased/Under the Hood-20241205-144036.yaml b/.changes/unreleased/Under the Hood-20241205-144036.yaml new file mode 100644 index 000000000..aedcb4ce1 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241205-144036.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Use new `batch` context variables over `node.config.__dbt_internal` ones +time: 2024-12-05T14:40:36.373637-05:00 +custom: + Author: michelleark + Issue: "1263" diff --git a/dbt/include/snowflake/macros/materializations/merge.sql b/dbt/include/snowflake/macros/materializations/merge.sql index 716a325c1..7515f6010 100644 --- a/dbt/include/snowflake/macros/materializations/merge.sql +++ b/dbt/include/snowflake/macros/materializations/merge.sql @@ -57,10 +57,10 @@ {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%} {#-- Add additional incremental_predicates to filter for batch --#} - {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} + {% if model.batch and model.batch.event_time_start -%} {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "')") %} {% endif %} - {% if model.config.__dbt_internal_microbatch_event_time_end -%} + {% if model.batch and model.batch.event_time_end -%} {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "')") %} {% endif %} {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} From b3eeb083ce379ee84468cd485e58ea10b353350a Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 9 Dec 2024 21:44:30 +0000 Subject: [PATCH 37/41] Fix refresh_strategy = auto semantics for dynamic tables (#1268) * Add failing test. * Parametrize cases * Change comparison logic for when to rebuild if DT has auto refresh strategy (implicit or explicit). * Add changelog --------- Co-authored-by: VersusFacit --- .../unreleased/Fixes-20241209-131530.yaml | 6 +++ dbt/adapters/snowflake/relation.py | 6 ++- .../snowflake/relation_configs/__init__.py | 1 + .../dynamic_table_tests/models.py | 20 +++++++++ .../dynamic_table_tests/test_basic.py | 43 ++++++++++++++++++- 5 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241209-131530.yaml diff --git a/.changes/unreleased/Fixes-20241209-131530.yaml b/.changes/unreleased/Fixes-20241209-131530.yaml new file mode 100644 index 000000000..b3196a3b0 --- /dev/null +++ b/.changes/unreleased/Fixes-20241209-131530.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: AUTO should no longer lead to rebuilds of dynamic tables. +time: 2024-12-09T13:15:30.554566-08:00 +custom: + Author: versusfacit + Issue: "1267" diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index b6924b9b3..54db21924 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -17,6 +17,7 @@ from dbt_common.events.functions import fire_event, warn_or_error from dbt.adapters.snowflake.relation_configs import ( + RefreshMode, SnowflakeCatalogConfigChange, SnowflakeDynamicTableConfig, SnowflakeDynamicTableConfigChangeset, @@ -109,7 +110,10 @@ def dynamic_table_config_changeset( ) ) - if new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode: + if ( + new_dynamic_table.refresh_mode != RefreshMode.AUTO + and new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode + ): config_change_collection.refresh_mode = SnowflakeDynamicTableRefreshModeConfigChange( action=RelationConfigChangeAction.create, context=new_dynamic_table.refresh_mode, diff --git a/dbt/adapters/snowflake/relation_configs/__init__.py b/dbt/adapters/snowflake/relation_configs/__init__.py index fec9d8a54..67f7644d2 100644 --- a/dbt/adapters/snowflake/relation_configs/__init__.py +++ b/dbt/adapters/snowflake/relation_configs/__init__.py @@ -3,6 +3,7 @@ SnowflakeCatalogConfigChange, ) from dbt.adapters.snowflake.relation_configs.dynamic_table import ( + RefreshMode, SnowflakeDynamicTableConfig, SnowflakeDynamicTableConfigChangeset, SnowflakeDynamicTableRefreshModeConfigChange, diff --git a/tests/functional/relation_tests/dynamic_table_tests/models.py b/tests/functional/relation_tests/dynamic_table_tests/models.py index 4dcd6cf48..57d83f968 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/models.py +++ b/tests/functional/relation_tests/dynamic_table_tests/models.py @@ -17,6 +17,26 @@ """ +EXPLICIT_AUTO_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='AUTO', +) }} +select * from {{ ref('my_seed') }} +""" + +IMPLICIT_AUTO_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', +) }} +select * from {{ ref('my_seed') }} +""" + + DYNAMIC_TABLE_DOWNSTREAM = """ {{ config( materialized='dynamic_table', diff --git a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py index 79a2241ca..8cdf59ebc 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py +++ b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py @@ -1,6 +1,6 @@ import pytest -from dbt.tests.util import run_dbt +from dbt.tests.util import assert_message_in_logs, run_dbt, run_dbt_and_capture from tests.functional.relation_tests.dynamic_table_tests import models from tests.functional.utils import query_relation_type @@ -46,3 +46,44 @@ class TestBasicIcebergOn(TestBasic): @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"enable_iceberg_materializations": True}} + + +class TestAutoConfigDoesntFullRefresh: + """ + AUTO refresh_strategy will be compared accurately with both INCREMENTAL and FULL. + https://github.com/dbt-labs/dbt-snowflake/issues/1267 + """ + + DT_NAME = "my_dynamic_table" + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + f"explicit_{self.DT_NAME}.sql": models.EXPLICIT_AUTO_DYNAMIC_TABLE, + f"implicit_{self.DT_NAME}.sql": models.IMPLICIT_AUTO_DYNAMIC_TABLE, + } + + @pytest.mark.parametrize("test_dt", [f"explicit_{DT_NAME}", f"implicit_{DT_NAME}"]) + def test_auto_config_doesnt_full_refresh(self, project, test_dt): + model_qualified_name = f"{project.database}.{project.test_schema}.{test_dt}" + + run_dbt(["seed"]) + _, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"]) + assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs) + assert_message_in_logs("refresh_mode = AUTO", logs) + + _, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"]) + + assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs, False) + assert_message_in_logs( + f"create or replace dynamic table {model_qualified_name}", logs, False + ) + assert_message_in_logs("refresh_mode = AUTO", logs, False) + assert_message_in_logs( + f"No configuration changes were identified on: `{model_qualified_name}`. Continuing.", + logs, + ) From efc68e4c32b58536f174e18ec28550bd9f88d629 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:17:05 -0500 Subject: [PATCH 38/41] Update default python version for python models (#1273) * update the default python version for python models; add a configuration file to collect future settings --- .changes/unreleased/Under the Hood-20241211-170831.yaml | 6 ++++++ dbt/adapters/snowflake/constants.py | 1 + dbt/adapters/snowflake/impl.py | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20241211-170831.yaml create mode 100644 dbt/adapters/snowflake/constants.py diff --git a/.changes/unreleased/Under the Hood-20241211-170831.yaml b/.changes/unreleased/Under the Hood-20241211-170831.yaml new file mode 100644 index 000000000..17bf42c1f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241211-170831.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Update default Python version for python models +time: 2024-12-11T17:08:31.842063-05:00 +custom: + Author: mikealfare + Issue: "1203" diff --git a/dbt/adapters/snowflake/constants.py b/dbt/adapters/snowflake/constants.py new file mode 100644 index 000000000..9c475dcd9 --- /dev/null +++ b/dbt/adapters/snowflake/constants.py @@ -0,0 +1 @@ +DEFAULT_PYTHON_VERSION_FOR_PYTHON_MODELS = "3.9" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 10ad2a8a1..7ccff9f8a 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -22,6 +22,7 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values +from dbt.adapters.snowflake import constants from dbt.adapters.snowflake.relation_configs import ( SnowflakeRelationType, TableFormat, @@ -345,7 +346,9 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): schema = parsed_model["schema"] database = parsed_model["database"] identifier = parsed_model["alias"] - python_version = parsed_model["config"].get("python_version", "3.8") + python_version = parsed_model["config"].get( + "python_version", constants.DEFAULT_PYTHON_VERSION_FOR_PYTHON_MODELS + ) packages = parsed_model["config"].get("packages", []) imports = parsed_model["config"].get("imports", []) From b8607e9d18c94d907657d3d6205a84906142700f Mon Sep 17 00:00:00 2001 From: "Andrew C. Hawkins" Date: Thu, 19 Dec 2024 18:07:39 -0500 Subject: [PATCH 39/41] Change how a column of type VECTOR is parsed (#1169) * Change how a column of type VECTOR is parsed * Add changelog entry --------- Co-authored-by: Andrew Hawkins Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .changes/unreleased/Fixes-20241018-173123.yaml | 6 ++++++ dbt/adapters/snowflake/column.py | 8 ++++++++ tests/unit/test_snowflake_adapter.py | 13 +++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 .changes/unreleased/Fixes-20241018-173123.yaml diff --git a/.changes/unreleased/Fixes-20241018-173123.yaml b/.changes/unreleased/Fixes-20241018-173123.yaml new file mode 100644 index 000000000..eab4e8376 --- /dev/null +++ b/.changes/unreleased/Fixes-20241018-173123.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix parsing of the VECTOR type +time: 2024-10-18T17:31:23.931299-04:00 +custom: + Author: achawkins + Issue: "1098" diff --git a/dbt/adapters/snowflake/column.py b/dbt/adapters/snowflake/column.py index de92a051f..281831b29 100644 --- a/dbt/adapters/snowflake/column.py +++ b/dbt/adapters/snowflake/column.py @@ -41,3 +41,11 @@ def string_size(self) -> int: return 16777216 else: return int(self.char_size) + + @classmethod + def from_description(cls, name: str, raw_data_type: str) -> "SnowflakeColumn": + if "vector" in raw_data_type.lower(): + column = cls(name, raw_data_type, None, None, None) + else: + column = super().from_description(name, raw_data_type) + return column diff --git a/tests/unit/test_snowflake_adapter.py b/tests/unit/test_snowflake_adapter.py index aa580aad2..18a5310dc 100644 --- a/tests/unit/test_snowflake_adapter.py +++ b/tests/unit/test_snowflake_adapter.py @@ -863,6 +863,19 @@ def test_float_from_description(self): assert col.is_string() is False assert col.is_integer() is False + def test_vector_from_description(self): + col = SnowflakeColumn.from_description("my_col", "VECTOR(FLOAT, 768)") + assert col.column == "my_col" + assert col.dtype == "VECTOR(FLOAT, 768)" + assert col.char_size is None + assert col.numeric_precision is None + assert col.numeric_scale is None + assert col.is_float() is False + assert col.is_number() is False + assert col.is_numeric() is False + assert col.is_string() is False + assert col.is_integer() is False + class SnowflakeConnectionsTest(unittest.TestCase): def test_comment_stripping_regex(self): From b687ac477a4e816ab5ea381a3685da1dc886ff87 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:02:31 -0500 Subject: [PATCH 40/41] ADAP-1115: Move to `hatch` and `pyproject.toml` (#1250) * move config files into pyproject.toml and hatch.toml * update workflow files to use hatch.toml * move test matrix into the workflow * fix pytest config in pyproject.toml --- .bumpversion.cfg | 35 -- .../Under the Hood-20241117-184430.yaml | 6 + .github/scripts/integration-test-matrix.js | 95 ---- .github/scripts/update_dependencies.sh | 15 - .../scripts/update_dev_dependency_branches.sh | 7 +- .github/workflows/integration.yml | 105 +---- .github/workflows/main.yml | 56 +-- .github/workflows/nightly-release.yml | 14 +- .github/workflows/release.yml | 108 ++--- .github/workflows/release_prep_hatch.yml | 440 ++++++++++++++++++ MANIFEST.in | 1 - Makefile | 48 -- dev-requirements.txt | 23 - hatch.toml | 62 +++ pyproject.toml | 48 ++ pytest.ini | 10 - setup.py | 81 ---- tox.ini | 32 -- 18 files changed, 635 insertions(+), 551 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .changes/unreleased/Under the Hood-20241117-184430.yaml delete mode 100644 .github/scripts/integration-test-matrix.js delete mode 100644 .github/scripts/update_dependencies.sh create mode 100644 .github/workflows/release_prep_hatch.yml delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 hatch.toml create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 1d774733d..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.10.0a1 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/snowflake/__version__.py] diff --git a/.changes/unreleased/Under the Hood-20241117-184430.yaml b/.changes/unreleased/Under the Hood-20241117-184430.yaml new file mode 100644 index 000000000..d2a7b67a6 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241117-184430.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move from setup.py to pyproject.toml and to hatch as a dev tool +time: 2024-11-17T18:44:30.85288-05:00 +custom: + Author: mikealfare + Issue: "1250" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index e2c88b00b..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,95 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.9"; - const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"]; - const supportedAdapters = ["snowflake"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const changes = JSON.parse(process.env.CHANGES); - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - console.log("changes", changes); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - if ( - changes.includes(adapter) || - testAllLabel || - labels.includes(`test ${adapter}`) - ) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-14", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-14"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh deleted file mode 100644 index c3df48e52..000000000 --- a/.github/scripts/update_dependencies.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index 022df6a8a..603dec17b 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -5,17 +5,20 @@ set -e dbt_adapters_branch=$1 dbt_core_branch=$2 dbt_common_branch=$3 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${dbt_adapters_branch}#subdirectory=dbt-tests-adapter|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$tests_req_sed_pattern" $target_req_file sed -i "" "$core_req_sed_pattern" $target_req_file sed -i "" "$common_req_sed_pattern" $target_req_file else sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$tests_req_sed_pattern" $target_req_file sed -i "$core_req_sed_pattern" $target_req_file sed -i "$common_req_sed_pattern" $target_req_file fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b3662d5c0..b939e2581 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -76,85 +76,31 @@ defaults: shell: bash jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v3 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - snowflake: - - '.github/**/*.yml' - - '.github/**/*.sh' - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - '*.py' - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v7 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} + name: ${{ matrix.os }} / python ${{ matrix.python-version }} # run if not a PR from a forked repository or has a label to mark as safe to test # also checks that the matrix generated is not empty if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && ( github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ok to test') ) runs-on: ${{ matrix.os }} - needs: test-metadata strategy: fail-fast: false - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} + matrix: + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + include: + - os: macos-14 + python-version: "3.9" + - os: windows-2022 + python-version: "3.9" env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} @@ -190,24 +136,19 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Update Adapters and Core branches (update dev_requirements.txt) + - name: Update Adapters and Core branches (update hatch.toml) if: ${{ github.event_name == 'workflow_dispatch' }} run: | ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} - cat dev-requirements.txt + cat hatch.toml - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version + uses: pypa/hatch@install - - name: Run tox (snowflake) - if: matrix.adapter == 'snowflake' + - run: hatch run integration-tests tests/functional --ddtrace env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} @@ -226,26 +167,6 @@ jobs: DBT_TEST_USER_1: dbt_test_role_1 DBT_TEST_USER_2: dbt_test_role_2 DBT_TEST_USER_3: dbt_test_role_3 - run: tox -- --ddtrace - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }} - path: ./logs - overwrite: true - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - overwrite: true require-label-comment: runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24d2fa60b..f545c9d62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,16 +52,8 @@ jobs: with: python-version: '3.9' - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - dbt --version - - name: Run pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure + uses: pre-commit/action@v3.0.1 unit: name: unit test / python ${{ matrix.python-version }} @@ -73,10 +65,6 @@ jobs: matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - steps: - name: Check out the repository uses: actions/checkout@v4 @@ -89,26 +77,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version + uses: pypa/hatch@install - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - overwrite: true + - run: hatch run unit-tests build: name: build packages @@ -129,25 +100,16 @@ jobs: with: python-version: '3.9' - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version + - uses: pypa/hatch@install - name: Build distributions - run: ./scripts/build-dist.sh + run: hatch build - name: Show distributions run: ls -lh dist/ - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 + run: hatch run build:check-all - name: Check if this is an alpha version id: check-is-alpha @@ -184,12 +146,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - uses: actions/download-artifact@v4 with: name: dist diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 16a5d0da1..17bf8102f 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -50,11 +50,15 @@ jobs: commit_sha=$(git rev-parse HEAD) echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - uses: pypa/hatch@install + + - id: version-number-sources + run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" id: semver diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad7cf76b4..e3e86c151 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,86 +119,70 @@ defaults: shell: bash jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Only Docker: ${{ inputs.only_docker }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + release-prep: + name: "Release prep: generate changelog, bump version" + uses: ./.github/workflows/release_prep_hatch.yml with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] + build-release: + name: "Build release" + needs: release-prep runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} steps: - - name: Print variables + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-prep.outputs.release-branch }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - uses: pypa/hatch@install + - id: archive run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + - run: hatch build && hatch run build:check-all + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.archive.outputs.name }} + path: dist/ + retention-days: 3 github-release: name: GitHub Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + needs: [build-release, release-prep] + uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} test_run: ${{ inputs.test_run }} + archive_name: ${{ needs.build-release.outputs.archive-name }} pypi-release: name: PyPI Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + needs: build-release + runs-on: ubuntu-latest + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + permissions: + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write + steps: + - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ needs.build-release.outputs.archive-name }} docker-release: name: "Docker Release" @@ -206,7 +190,7 @@ jobs: # what we need to release but draft releases don't actually tag the commit so it # finds nothing to release if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} - needs: [bump-version-generate-changelog, build-test-package, github-release] + needs: github-release permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml new file mode 100644 index 000000000..de782ad1a --- /dev/null +++ b/.github/workflows/release_prep_hatch.yml @@ -0,0 +1,440 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# branch: The branch that we will release from +# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# deploy-to: If we are deploying to prod or test, if test then release from branch +# is-nightly-release: Identifier that this is nightly release +# +# Outputs: +# release-sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +name: "Release prep" +run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +on: + workflow_call: + inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The version to release" + required: true + type: string + deploy-to: + description: "Deploy to test or prod" + type: string + default: "prod" + is-nightly-release: + description: "Identify if this is a nightly release" + type: boolean + default: false + outputs: + release-branch: + description: "The branch to be released from" + value: ${{ jobs.release.outputs.branch }} + release-sha: + description: "The SHA to be released" + value: ${{ jobs.release.outputs.sha }} + changelog-path: + description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" + value: ${{ jobs.release-inputs.outputs.changelog-path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.12 + NOTIFICATION_PREFIX: "[Release Prep]" + +jobs: + release-inputs: + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.changelog.outputs.path }} + changelog-exists: ${{ steps.changelog.outputs.exists }} + base-version: ${{ steps.semver.outputs.base-version }} + pre-release: ${{ steps.semver.outputs.pre-release }} + is-pre-release: ${{ steps.semver.outputs.is-pre-release }} + version-is-current: ${{ steps.version.outputs.is-current }} + + steps: + - name: "[DEBUG] Log inputs" + run: | + # WORKFLOW INPUTS + echo Branch: ${{ inputs.branch }} + echo Release version: ${{ inputs.version }} + echo Deploy to: ${{ inputs.deploy-to }} + echo Nightly release: ${{ inputs.is-nightly-release }} + # ENVIRONMENT VARIABLES + echo Python version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Parse input version" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ inputs.version }} + + - name: "Audit version" + id: version + run: | + is_current=false + current_version=$(hatch version) + if test "$current_version" = "${{ inputs.version }}" + then + is_current=true + fi + echo "is-current=$is_current" >> $GITHUB_OUTPUT + + - name: "[INFO] Skip version bump" + if: steps.version.outputs.is-current == 'true' + run: | + title="Skip version bump" + message="The version matches the input version ${{ inputs.version }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Audit changelog" + id: changelog + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + does_exist=false + if test -f $path + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[INFO] Skip changelog generation" + if: steps.changelog.outputs.exists == 'true' + run: | + title="Skip changelog generation" + message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-branch: + runs-on: ubuntu-latest + needs: release-inputs + if: | + needs.release-inputs.outputs.changelog-exists == 'false' || + needs.release-inputs.outputs.version-is-current == 'false' + outputs: + name: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Set release branch" + id: release-branch + run: | + name="prep-release/" + if [[ ${{ inputs.is-nightly-release }} == true ]] + then + name+="nightly-release/" + else + name+="${{ inputs.deploy-to }}/" + fi + name+="${{ inputs.version }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Create release branch ${{ steps.release-branch.outputs.name }}" + run: | + git checkout -b ${{ steps.release-branch.outputs.name }} + git push -u origin ${{ steps.release-branch.outputs.name }} + + - name: "[INFO] Create release branch" + run: | + title="Create release branch" + message="Create release branch: ${{ steps.release-branch.outputs.name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + core-team: + if: needs.release-inputs.outputs.changelog-exists == 'false' + needs: release-inputs + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + secrets: inherit + + generate-changelog: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.changelog-exists == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - core-team + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Install `changie`" + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --move-dir '${{ needs.release-inputs.outputs.base-version }}' \ + --prerelease ${{ needs.release-inputs.outputs.pre-release }} + elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --include '${{ needs.release-inputs.outputs.base-version }}' \ + --remove-prereleases + else # releasing a final patch with no pre-releases + changie batch ${{ needs.release-inputs.outputs.base-version }} + fi + changie merge + env: + CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }} + + - name: "Remove trailing whitespace and missing new lines" + # this step will fail on whitespace errors but also correct them + continue-on-error: true + run: hatch run code-quality + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + + - name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + title="Changelog generation" + if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]] + then + message="Generated changelog file successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Failed to generate changelog file" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + bump-version: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.version-is-current == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - generate-changelog + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Bump version to ${{ inputs.version }}" + run: hatch version ${{ inputs.version }} + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version to ${{ inputs.version }}" + run: | + title="Version bump" + message="Bumped version to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + unit-tests: + runs-on: ubuntu-latest + # only run unit tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run unit tests" + run: hatch run unit-tests + + integration-tests: + runs-on: ubuntu-latest + # only run integration tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }}\ + + - name: "Set up `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run integration tests" + run: hatch run integration-tests + + merge-release-branch: + runs-on: ubuntu-latest + needs: + - unit-tests + - integration-tests + - release-branch + - release-inputs + if: | + !failure() && !cancelled() && + needs.release-branch.result == 'success' && + inputs.deploy-to == 'prod' + + steps: + - name: "Checkout ${{ github.event.repository.name }}" + uses: actions/checkout@v4 + + - name: "Merge changes into ${{ inputs.branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.release-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[INFO] Merge changes into ${{ inputs.branch }}" + run: | + title="Merge changes" + message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release: + runs-on: ubuntu-latest + needs: + - release-branch + - merge-release-branch + if: ${{ !failure() && !cancelled() }} + + # Get the SHA that will be released. + # If the changelog already exists and the version was already current on the input branch, then release from there. + # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + branch: ${{ steps.branch.outputs.name }} + sha: ${{ steps.sha.outputs.sha }} + + steps: + - name: "Set release branch" + id: branch + # If a release branch was created and not merged, use the release branch + # Otherwise, use the input branch because either nothing was done, or the changes were merged back in + run: | + if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then + branch="${{ needs.release-branch.outputs.name }}" + else + branch="${{ inputs.branch }}" + fi + echo "name=$branch" >> $GITHUB_OUTPUT + + - name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.branch.outputs.name }} + + - name: "Set release SHA" + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # if this is a real release and a release branch was created, delete it + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index 9a2376f47..000000000 --- a/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-snowflake - -.PHONY: unit -unit: ## Runs unit tests with py39. - @\ - tox -e py39 - -.PHONY: test -test: ## Runs unit tests with py39 and code checks against staged changes. - @\ - tox -p -e py39; \ - pre-commit run --all-files - -.PHONY: integration -integration: ## Runs snowflake integration tests with py38. - @\ - tox -e py39-snowflake -- - -.PHONY: clean - @echo "cleaning repo" - @git clean -f -X - -.PHONY: help -help: ## Show this help message. - @echo 'usage: make [target]' - @echo - @echo 'targets:' - @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -.PHONY: docker-dev -docker-dev: - docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . - docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev - -.PHONY: docker-prod -docker-prod: - docker build -f docker/Dockerfile -t dbt-snowflake . diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 906003768..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# install latest changes in dbt-core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git - -# dev -ipdb~=0.13.13 -pre-commit~=3.7.0 - -# test -ddtrace==2.3.0 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.6 -tox~=4.16 - -# build -bumpversion~=0.6.0 -twine~=5.1 -wheel~=0.43 diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 000000000..2377e5d6c --- /dev/null +++ b/hatch.toml @@ -0,0 +1,62 @@ +[version] +path = "dbt/adapters/snowflake/__version__.py" + +[build.targets.sdist] +packages = ["src/dbt"] +sources = ["src"] + +[build.targets.wheel] +packages = ["src/dbt"] +sources = ["src"] + +[envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "ddtrace==2.3.0", + "ipdb~=0.13.13", + "pre-commit~=3.7.0", + "pytest~=7.4", + "pytest-csv~=3.0", + "pytest-dotenv~=0.5.2", + "pytest-logbook~=1.2", + "pytest-xdist~=3.6", + "tox~=4.16", # does this pin deps transitively? +] + +[envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "- python -m pytest {args:tests/functional}" +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .", + "docker run --rm -it --name dbt-snowflake-dev -v $(pwd):/opt/code dbt-snowflake-dev", +] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] + +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt_snowflake-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-snowflake", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_snowflake-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-snowflake", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-snowflake ." diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..568aa3533 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-snowflake" +description = "The Snowflake adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "snowflake"] +requires-python = ">=3.9.0" +authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "dbt-common>=1.10,<2.0", + "dbt-adapters>=1.10.4,<2.0", + "snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0", + # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "agate", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-snowflake" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-snowflake.git" +Issues = "https://github.com/dbt-labs/dbt-snowflake/issues" +Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" + +[tool.pytest.ini_options] +testpaths = ["tests/functional", "tests/unit"] +env_files = ["test.env"] +addopts = "-v --color=yes -n auto" +filterwarnings = [ + "ignore:datetime.datetime.utcnow:DeprecationWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b04a6ccf3..000000000 --- a/pytest.ini +++ /dev/null @@ -1,10 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/integration - tests/functional diff --git a/setup.py b/setup.py deleted file mode 100644 index b237182ce..000000000 --- a/setup.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -import os -from pathlib import Path - -import sys - -# require python 3.8 or newer -if sys.version_info < (3, 9): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.9 or higher.") - sys.exit(1) - - -# require version of setuptools that supports find_namespace_packages -from setuptools import setup - -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again") - sys.exit(1) - - -# pull long description from README -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, "README.md")) as f: - long_description = f.read() - - -# used for this adapter's version -VERSION = Path(__file__).parent / "dbt/adapters/snowflake/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -package_name = "dbt-snowflake" -description = """The Snowflake adapter plugin for dbt""" - -setup( - name=package_name, - version=_plugin_version(), - description=description, - long_description=long_description, - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-snowflake", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.10.4,<2.0", - "snowflake-connector-python[secure-local-storage]~=3.0", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0", - # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "agate", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - ], - python_requires=">=3.9", -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index f6952efaf..000000000 --- a/tox.ini +++ /dev/null @@ -1,32 +0,0 @@ -[tox] -skipsdist = True -envlist = py39,py310,py311,py312 - -[testenv:{unit,py39,py310,py311,py312,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - SNOWFLAKE_TEST_* - PYTEST_ADDOPTS - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - snowflake: {envpython} -m pytest {posargs} tests/functional -deps = - -rdev-requirements.txt - -e. From 5d935eedbac8199e5fbf4022d291abfba8198608 Mon Sep 17 00:00:00 2001 From: Leah Procopi Date: Fri, 24 Jan 2025 11:29:16 -0800 Subject: [PATCH 41/41] Feature: Custom Iceberg `base_location_root` (#1289) * update relation & add tests * switch if/else pattern * add change log * add test for dynamic table with path and subpath * modify base_location config * update to base_location_root, include schema and table name in path * resolve linting error (f-string nested double quotes) * resolve linting error (move to new line) * add unit tests & make final edits to functional tests * lint unit test * update changie * update dt test cases for iceberg to be dynamic --------- Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Features-20250113-133414.yaml | 6 ++ dbt/adapters/snowflake/impl.py | 1 + dbt/adapters/snowflake/relation.py | 5 +- tests/functional/iceberg/models.py | 63 +++++++++++++++ tests/functional/iceberg/test_table_basic.py | 12 ++- tests/unit/test_iceberg_location.py | 79 +++++++++++++++++++ 6 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Features-20250113-133414.yaml create mode 100644 tests/unit/test_iceberg_location.py diff --git a/.changes/unreleased/Features-20250113-133414.yaml b/.changes/unreleased/Features-20250113-133414.yaml new file mode 100644 index 000000000..869ed6b17 --- /dev/null +++ b/.changes/unreleased/Features-20250113-133414.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Added support for custom iceberg base_location_root +time: 2025-01-13T13:34:14.326047-08:00 +custom: + Author: LProcopi15 + Issue: "1284" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 7ccff9f8a..6ae8ef183 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -54,6 +54,7 @@ class SnowflakeConfig(AdapterConfig): # extended formats table_format: Optional[str] = None external_volume: Optional[str] = None + base_location_root: Optional[str] = None base_location_subpath: Optional[str] = None diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 54db21924..f3ee3e510 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -204,7 +204,10 @@ def get_ddl_prefix_for_alter(self) -> str: return "" def get_iceberg_ddl_options(self, config: RelationConfig) -> str: - base_location: str = f"_dbt/{self.schema}/{self.name}" + # If the base_location_root config is supplied, overwrite the default value ("_dbt/") + base_location: str = ( + f"{config.get('base_location_root', '_dbt')}/{self.schema}/{self.name}" + ) if subpath := config.get("base_location_subpath"): base_location += f"/{subpath}" diff --git a/tests/functional/iceberg/models.py b/tests/functional/iceberg/models.py index 6433f74bf..e6da6aca4 100644 --- a/tests/functional/iceberg/models.py +++ b/tests/functional/iceberg/models.py @@ -23,6 +23,37 @@ select * from {{ ref('first_table') }} """ +_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH = """ +{{ + config( + transient = "true", + materialized = "table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + base_location_subpath="subpath", + ) +}} + +select * from {{ ref('first_table') }} +""" + _MODEL_BASIC_DYNAMIC_TABLE_MODEL = """ {{ config( materialized='dynamic_table', @@ -36,6 +67,38 @@ select * from {{ ref('first_table') }} """ +_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH = """ +{{ + config( + transient = "transient", + materialized = "dynamic_table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + ) +}} + +select * from {{ ref('first_table') }} +""" + +_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH = """ +{{ + config( + transient = "true", + materialized = "dynamic_table", + cluster_by=['id'], + table_format="iceberg", + external_volume="s3_iceberg_snow", + base_location_root="root_path", + base_location_subpath='subpath', + ) +}} + +select * from {{ ref('first_table') }} +""" + + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH = """ {{ config( materialized='dynamic_table', diff --git a/tests/functional/iceberg/test_table_basic.py b/tests/functional/iceberg/test_table_basic.py index e835a5fce..faf4b34f7 100644 --- a/tests/functional/iceberg/test_table_basic.py +++ b/tests/functional/iceberg/test_table_basic.py @@ -7,7 +7,11 @@ from tests.functional.iceberg.models import ( _MODEL_BASIC_TABLE_MODEL, _MODEL_BASIC_ICEBERG_MODEL, + _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH, + _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH, _MODEL_BASIC_DYNAMIC_TABLE_MODEL, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH, + _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH, _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, _MODEL_BUILT_ON_ICEBERG_TABLE, _MODEL_TABLE_BEFORE_SWAP, @@ -26,14 +30,18 @@ def models(self): return { "first_table.sql": _MODEL_BASIC_TABLE_MODEL, "iceberg_table.sql": _MODEL_BASIC_ICEBERG_MODEL, + "iceberg_tableb.sql": _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH, + "iceberg_tablec.sql": _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH, "table_built_on_iceberg_table.sql": _MODEL_BUILT_ON_ICEBERG_TABLE, "dynamic_table.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL, - "dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, + "dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH, + "dynamic_tablec.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH, + "dynamic_tabled.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH, } def test_iceberg_tables_build_and_can_be_referred(self, project): run_results = run_dbt() - assert len(run_results) == 5 + assert len(run_results) == 9 class TestIcebergTableTypeBuildsOnExistingTable: diff --git a/tests/unit/test_iceberg_location.py b/tests/unit/test_iceberg_location.py new file mode 100644 index 000000000..dca82b47e --- /dev/null +++ b/tests/unit/test_iceberg_location.py @@ -0,0 +1,79 @@ +import pytest +from dbt.adapters.snowflake.relation import SnowflakeRelation + + +@pytest.fixture +def iceberg_config() -> dict: + """Fixture providing standard Iceberg configuration.""" + return { + "schema": "my_schema", + "identifier": "my_table", + "external_volume": "s3_iceberg_snow", + "base_location_root": "root_path", + "base_location_subpath": "subpath", + } + + +def get_actual_base_location(config: dict[str, str]) -> str: + """Get the actual base location from the configuration by parsing the DDL predicates.""" + + relation = SnowflakeRelation.create( + schema=config["schema"], + identifier=config["identifier"], + ) + + actual_ddl_predicates = relation.get_iceberg_ddl_options(config).strip() + actual_base_location = actual_ddl_predicates.split("base_location = ")[1] + + return actual_base_location + + +def test_iceberg_path_and_subpath(iceberg_config: dict[str, str]): + """Test when base_location_root and base_location_subpath are provided""" + expected_base_location = ( + f"'{iceberg_config['base_location_root']}/" + f"{iceberg_config['schema']}/" + f"{iceberg_config['identifier']}/" + f"{iceberg_config['base_location_subpath']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location + + +def test_iceberg_only_subpath(iceberg_config: dict[str, str]): + """Test when only base_location_subpath is provided""" + del iceberg_config["base_location_root"] + + expected_base_location = ( + f"'_dbt/" + f"{iceberg_config['schema']}/" + f"{iceberg_config['identifier']}/" + f"{iceberg_config['base_location_subpath']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location + + +def test_iceberg_only_path(iceberg_config: dict[str, str]): + """Test when only base_location_root is provided""" + del iceberg_config["base_location_subpath"] + + expected_base_location = ( + f"'{iceberg_config['base_location_root']}/" + f"{iceberg_config['schema']}/" + f"{iceberg_config['identifier']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location + + +def test_iceberg_no_path(iceberg_config: dict[str, str]): + """Test when no base_location_root or is base_location_subpath provided""" + del iceberg_config["base_location_root"] + del iceberg_config["base_location_subpath"] + + expected_base_location = ( + f"'_dbt/" f"{iceberg_config['schema']}/" f"{iceberg_config['identifier']}'" + ).strip() + + assert get_actual_base_location(iceberg_config) == expected_base_location