diff --git a/.changes/unreleased/Fixes-20240628-190140.yaml b/.changes/unreleased/Fixes-20240628-190140.yaml new file mode 100644 index 000000000..c58b465fd --- /dev/null +++ b/.changes/unreleased/Fixes-20240628-190140.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix scenario where using the `--empty` flag causes metadata queries to contain + limit clauses +time: 2024-06-28T19:01:40.558234-04:00 +custom: + Author: mikealfare + Issue: "1033" diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index b858aec11..177720486 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -27,14 +27,14 @@ {% macro snowflake__get_columns_in_relation(relation) -%} {%- set sql -%} - describe table {{ relation }} + describe table {{ relation.render() }} {%- endset -%} {%- set result = run_query(sql) -%} {% set maximum = 10000 %} {% if (result | length) >= maximum %} {% set msg %} - Too many columns in relation {{ relation }}! dbt can only get + Too many columns in relation {{ relation.render() }}! dbt can only get information about relations with fewer than {{ maximum }} columns. {% endset %} {% do exceptions.raise_compiler_error(msg) %} @@ -186,7 +186,7 @@ {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} {% call statement('alter_column_type') %} - alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; + alter table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; {% endcall %} {% endmacro %} @@ -196,7 +196,7 @@ {%- else -%} {%- set relation_type = relation.type -%} {%- endif -%} - comment on {{ relation_type }} {{ relation }} IS $${{ relation_comment | replace('$', '[$]') }}$$; + comment on {{ relation_type }} {{ relation.render() }} IS $${{ relation_comment | replace('$', '[$]') }}$$; {% endmacro %} @@ -207,7 +207,7 @@ {% else -%} {% set relation_type = relation.type %} {% endif %} - alter {{ relation_type }} {{ relation }} alter + alter {{ relation_type }} {{ relation.render() }} alter {% for column_name in existing_columns if (column_name in existing_columns) or (column_name|lower in existing_columns) %} {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last else ';' }} {% endfor %} @@ -266,7 +266,7 @@ {% if add_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation }} add column + alter {{ relation_type }} {{ relation.render() }} add column {% for column in add_columns %} {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %} @@ -279,7 +279,7 @@ {% if remove_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation }} drop column + alter {{ relation_type }} {{ relation.render() }} drop column {% for column in remove_columns %} {{ column.name }}{{ ',' if not loop.last }} {% endfor %} @@ -312,7 +312,7 @@ {% macro snowflake__truncate_relation(relation) -%} {% set truncate_dml %} - truncate table {{ relation }} + truncate table {{ relation.render() }} {% endset %} {% call statement('truncate_relation') -%} {{ snowflake_dml_explicit_transaction(truncate_dml) }} diff --git a/tests/functional/adapter/empty/_models.py b/tests/functional/adapter/empty/_models.py new file mode 100644 index 000000000..698e4f1db --- /dev/null +++ b/tests/functional/adapter/empty/_models.py @@ -0,0 +1,72 @@ +SEED = """ +my_id,my_value +1,a +2,b +3,c +""".strip() + + +SCHEMA = """ +version: 2 + +seeds: + - name: my_seed + description: "This is my_seed" + columns: + - name: id + description: "This is my_seed.my_id" +""" + +CONTROL = """ +select * from {{ ref("my_seed") }} +""" + + +GET_COLUMNS_IN_RELATION = """ +{{ config(materialized="table") }} +{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_TYPE = """ +{{ config(materialized="table") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"relations": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"columns": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_ADD_REMOVE_COLUMNS = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{% set my_column = api.Column("my_column", "varchar") %} +{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %} +{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %} +select * from {{ ref("my_seed") }} +""" + + +TRUNCATE_RELATION = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{{ truncate_relation(my_seed) }} +select * from {{ ref("my_seed") }} +""" diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index 401179b71..fe07fc081 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -1,4 +1,8 @@ from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.empty import _models class TestSnowflakeEmpty(BaseTestEmpty): @@ -7,3 +11,41 @@ class TestSnowflakeEmpty(BaseTestEmpty): class TestSnowflakeEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): pass + + +class TestMetadataWithEmptyFlag: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": _models.SEED} + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": _models.SCHEMA, + "control.sql": _models.CONTROL, + "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, + "alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT, + "alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT, + "alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS, + "truncate_relation.sql": _models.TRUNCATE_RELATION, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + + @pytest.mark.parametrize( + "model", + [ + "control", + "get_columns_in_relation", + "alter_column_type", + "alter_relation_comment", + "alter_column_comment", + "alter_relation_add_remove_columns", + "truncate_relation", + ], + ) + def test_run(self, project, model): + run_dbt(["run", "--empty", "--select", model]) diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 51346124f..86ea0a346 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -1,4 +1,5 @@ import pytest +import uuid from dbt.tests.util import run_dbt, write_file from dbt.tests.adapter.python_model.test_python_model import ( BasePythonModelTests, @@ -174,20 +175,24 @@ def test_external_access_integration(self, project): run_dbt(["run"]) -SECRETS_MODE = """ +TEST_RUN_ID = uuid.uuid4().hex +TEST_SECRET = f"test_secret_{TEST_RUN_ID}" +TEST_NETWORK_RULE = f"test_network_rule_{TEST_RUN_ID}" +TEST_EXTERNAL_ACCESS_INTEGRATION = f"test_external_access_integration_{TEST_RUN_ID}" +SECRETS_MODE = f""" import pandas import snowflake.snowpark as snowpark def model(dbt, session: snowpark.Session): dbt.config( materialized="table", - secrets={"secret_variable_name": "test_secret"}, - external_access_integrations=["test_external_access_integration"], + secrets={{"secret_variable_name": "{TEST_SECRET}"}}, + external_access_integrations=["{TEST_EXTERNAL_ACCESS_INTEGRATION}"], ) import _snowflake return session.create_dataframe( pandas.DataFrame( - [{"secret_value": _snowflake.get_generic_secret_string('secret_variable_name')}] + [{{"secret_value": _snowflake.get_generic_secret_string('secret_variable_name')}}] ) ) """ @@ -198,23 +203,29 @@ class TestSecrets: def models(self): return {"secret_python_model.py": SECRETS_MODE} - # This test can be flaky because of delays in the integration being set up before trying to use it. @pytest.fixture(scope="class") def profiles_config_update(self): return {"retry_all": True, "connect_retries": 3} def test_secrets(self, project): project.run_sql( - "create or replace secret test_secret type = generic_string secret_string='secret value';" + f"create or replace secret {TEST_SECRET} type = generic_string secret_string='secret value';" ) - # The secrets you specify as values must also be specified in the external access integration. - # See https://docs.snowflake.com/en/developer-guide/external-network-access/creating-using-external-network-access#using-the-external-access-integration-in-a-function-or-procedure - project.run_sql( - "create or replace network rule test_network_rule type = host_port mode = egress value_list= ('www.google.com:443');" + f"create or replace network rule {TEST_NETWORK_RULE} type = host_port mode = egress value_list= ('www.google.com:443');" ) + project.run_sql( - "create or replace external access integration test_external_access_integration allowed_network_rules = (test_network_rule) allowed_authentication_secrets = (test_secret) enabled = true;" + f"create or replace external access integration {TEST_EXTERNAL_ACCESS_INTEGRATION} " + f"allowed_network_rules = ({TEST_NETWORK_RULE}) " + f"allowed_authentication_secrets = ({TEST_SECRET}) enabled = true;" ) + run_dbt(["run"]) + + project.run_sql(f"drop secret if exists {TEST_SECRET};") + project.run_sql(f"drop network rule if exists {TEST_NETWORK_RULE};") + project.run_sql( + f"drop external access integration if exists {TEST_EXTERNAL_ACCESS_INTEGRATION};" + )