From 4b2d93808e1b852aec3b1c281fa0af231b0cf297 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:40:47 -0600 Subject: [PATCH] Migrate to dbt-common and dbt-adapters package (#885) * update RELEASE_BRANCH env * create draft pr to track work * initial migration work * get tests_connections unit test passing * update model_node refs to relation_config, update _macro_manifest_lazy ref in unit test * update new_config ping in dynamic_table configuration_changes macro * minor changes to imports * minor changes, revert dev-requirements pointer * revert runtime_config call * fix list_relations_without_caching * change up test input for schema_relation * update snowflake__get_paginated_relations_array macro schema_relation call * add changelog * prepare draft pr for dbt-common name change in core * change pointer and add ref to dbt-common in setup.py * change pointer back to main post merge in core * adjust one nit import and ref * add dbt-adapters and changelog * add dbt-adapters * migrate to dbt-adapters package * slight change to dbt-adapter pointer, and slight change in fixture ref for dbt_test on test_column_types * swap dbt-adapters ref in setup.py * try and pin common and adapters * update fixtures * update dbt-adapters ref --------- Co-authored-by: Colin --- .../Under the Hood-20240117-112026.yaml | 6 ++++++ dbt/adapters/snowflake/column.py | 2 +- dbt/adapters/snowflake/connections.py | 15 +++++++------- dbt/adapters/snowflake/impl.py | 6 +++--- .../snowflake/relation_configs/policies.py | 2 +- setup.py | 2 ++ .../adapter/column_types/fixtures.py | 2 +- .../test_dynamic_tables_changes.py | 2 +- .../simple_seed/test_simple_seed_override.py | 20 +++++++++---------- tests/functional/query_tag/test_query_tags.py | 2 +- tests/unit/test_snowflake_adapter.py | 2 +- tests/unit/utils.py | 4 ++-- 12 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240117-112026.yaml diff --git a/.changes/unreleased/Under the Hood-20240117-112026.yaml b/.changes/unreleased/Under the Hood-20240117-112026.yaml new file mode 100644 index 000000000..edf040221 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240117-112026.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Migrate to dbt-common and dbt-adapters package +time: 2024-01-17T11:20:26.713584-06:00 +custom: + Author: McKnight-42 + Issue: "890" diff --git a/dbt/adapters/snowflake/column.py b/dbt/adapters/snowflake/column.py index 61e37f6cc..de92a051f 100644 --- a/dbt/adapters/snowflake/column.py +++ b/dbt/adapters/snowflake/column.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from dbt.adapters.base.column import Column -from dbt.common.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtRuntimeError @dataclass diff --git a/dbt/adapters/snowflake/connections.py b/dbt/adapters/snowflake/connections.py index f2f0bddd6..6eea0c4e4 100644 --- a/dbt/adapters/snowflake/connections.py +++ b/dbt/adapters/snowflake/connections.py @@ -11,7 +11,7 @@ from typing import Optional, Tuple, Union, Any, List import agate -import dbt.common.clients.agate_helper +from dbt_common.clients.agate_helper import empty_table from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -32,20 +32,19 @@ BindUploadError, ) -from dbt.common.exceptions import ( +from dbt_common.exceptions import ( DbtInternalError, DbtRuntimeError, DbtConfigError, ) -from dbt.common.exceptions import DbtDatabaseError -from dbt.adapters.base import Credentials # type: ignore +from dbt_common.exceptions import DbtDatabaseError from dbt.adapters.exceptions.connection import FailedToConnectError -from dbt.adapters.contracts.connection import AdapterResponse, Connection +from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials from dbt.adapters.sql import SQLConnectionManager # type: ignore from dbt.adapters.events.logging import AdapterLogger # type: ignore -from dbt.common.events.functions import warn_or_error +from dbt_common.events.functions import warn_or_error from dbt.adapters.events.types import AdapterEventWarning -from dbt.common.ui import line_wrap_message, warning_tag +from dbt_common.ui import line_wrap_message, warning_tag logger = AdapterLogger("Snowflake") @@ -476,7 +475,7 @@ def execute( if fetch: table = self.get_result_from_cursor(cursor, limit) else: - table = dbt.common.clients.agate_helper.empty_table() + table = empty_table() return response, table def add_standard_query(self, sql: str, **kwargs) -> Tuple[Connection, Any]: diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 40b54b61b..4dc57a1b4 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -15,9 +15,9 @@ from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation from dbt.adapters.snowflake import SnowflakeColumn -from dbt.common.contracts.constraints import ConstraintType -from dbt.common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError -from dbt.common.utils import filter_null_values +from dbt_common.contracts.constraints import ConstraintType +from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError +from dbt_common.utils import filter_null_values @dataclass diff --git a/dbt/adapters/snowflake/relation_configs/policies.py b/dbt/adapters/snowflake/relation_configs/policies.py index f0872f992..75195f9a3 100644 --- a/dbt/adapters/snowflake/relation_configs/policies.py +++ b/dbt/adapters/snowflake/relation_configs/policies.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from dbt.adapters.base.relation import Policy -from dbt.common.dataclass_schema import StrEnum +from dbt_common.dataclass_schema import StrEnum class SnowflakeRelationType(StrEnum): diff --git a/setup.py b/setup.py index 0eaf4fb9a..a9e1149db 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,8 @@ def _get_dbt_core_version(): include_package_data=True, install_requires=[ "dbt-core~={}".format(dbt_core_version), + "dbt-common~=0.1.0", + "dbt-adapters~=0.1.0a2", "snowflake-connector-python[secure-local-storage]~=3.0", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core "agate", diff --git a/tests/functional/adapter/column_types/fixtures.py b/tests/functional/adapter/column_types/fixtures.py index 293a03057..64b88111f 100644 --- a/tests/functional/adapter/column_types/fixtures.py +++ b/tests/functional/adapter/column_types/fixtures.py @@ -23,7 +23,7 @@ version: 2 models: - name: model - tests: + data_tests: - is_type: column_map: smallint_col: ['numeric', 'number', 'not string', 'not float', 'not integer'] diff --git a/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py b/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py index 17d14ebd8..984454fb9 100644 --- a/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py +++ b/tests/functional/adapter/dynamic_table_tests/test_dynamic_tables_changes.py @@ -2,7 +2,7 @@ import pytest -from dbt.common.contracts.config.materialization import OnConfigurationChangeOption +from dbt_common.contracts.config.materialization import OnConfigurationChangeOption from dbt.tests.util import ( assert_message_in_logs, get_model_file, diff --git a/tests/functional/adapter/simple_seed/test_simple_seed_override.py b/tests/functional/adapter/simple_seed/test_simple_seed_override.py index 4aed714ad..4dc559854 100644 --- a/tests/functional/adapter/simple_seed/test_simple_seed_override.py +++ b/tests/functional/adapter/simple_seed/test_simple_seed_override.py @@ -8,46 +8,46 @@ - name: seed_enabled columns: - name: birthday - tests: + data_tests: - column_type: type: character varying(16777216) - name: seed_id - tests: + data_tests: - column_type: type: FLOAT - name: seed_tricky columns: - name: seed_id - tests: + data_tests: - column_type: type: NUMBER(38,0) - name: seed_id_str - tests: + data_tests: - column_type: type: character varying(16777216) - name: a_bool - tests: + data_tests: - column_type: type: BOOLEAN - name: looks_like_a_bool - tests: + data_tests: - column_type: type: character varying(16777216) - name: a_date - tests: + data_tests: - column_type: type: TIMESTAMP_NTZ - name: looks_like_a_date - tests: + data_tests: - column_type: type: character varying(16777216) - name: relative - tests: + data_tests: - column_type: type: character varying(16777216) - name: weekday - tests: + data_tests: - column_type: type: character varying(16777216) """.lstrip() diff --git a/tests/functional/query_tag/test_query_tags.py b/tests/functional/query_tag/test_query_tags.py index 996a6a0c9..421aae1b6 100644 --- a/tests/functional/query_tag/test_query_tags.py +++ b/tests/functional/query_tag/test_query_tags.py @@ -31,7 +31,7 @@ - name: view_model_query_tag columns: - name: id - tests: + data_tests: - unique """ diff --git a/tests/unit/test_snowflake_adapter.py b/tests/unit/test_snowflake_adapter.py index 19f2165d1..a9f8f3572 100644 --- a/tests/unit/test_snowflake_adapter.py +++ b/tests/unit/test_snowflake_adapter.py @@ -13,7 +13,7 @@ from dbt.context.manifest import generate_query_header_context from dbt.context.providers import generate_runtime_macro_context from dbt.contracts.graph.manifest import ManifestStateCheck -from dbt.common.clients import agate_helper +from dbt_common.clients import agate_helper from snowflake import connector as snowflake_connector from .utils import ( diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 042e24bf2..b2b308185 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -9,7 +9,7 @@ import agate import pytest -from dbt.common.dataclass_schema import ValidationError +from dbt_common.dataclass_schema import ValidationError from dbt.config.project import PartialProject @@ -230,7 +230,7 @@ def assert_fails_validation(dct, cls): class TestAdapterConversions(TestCase): def _get_tester_for(self, column_type): - from dbt.common.clients import agate_helper + from dbt_common.clients import agate_helper if column_type is agate.TimeDelta: # dbt never makes this! return agate.TimeDelta()