From 032b5b586ae5cd3e22b0559c546fb77bec7ac6a6 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 14 Apr 2023 15:33:14 -0500 Subject: [PATCH 1/9] update RELEASE_BRANCH env --- .github/workflows/nightly-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 54c5fdc69..4762d1218 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -26,7 +26,7 @@ defaults: shell: bash env: - RELEASE_BRANCH: "1.4.latest" + RELEASE_BRANCH: "1.5.latest" jobs: aggregate-release-data: From a35b9ccb3564c6dd8f907cf6509e353d6418283a Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 19 Mar 2024 16:34:41 -0500 Subject: [PATCH 2/9] initial push to fix dynamic tables now that they have BASE TABLE as object type --- dbt/include/snowflake/macros/catalog.sql | 5 ++++- tests/functional/adapter/dynamic_table_tests/utils.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index f0c766865..e704771cb 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -41,7 +41,10 @@ table_catalog as "table_database", table_schema as "table_schema", table_name as "table_name", - coalesce(table_type, 'DYNAMIC TABLE') as "table_type", + case + when target_lag not null and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + else coalesce(table_type, 'BASE TABLE') + end as "table_type", comment as "table_comment", -- note: this is the _role_ that owns the table diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py index 1f145ec04..87196780c 100644 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ b/tests/functional/adapter/dynamic_table_tests/utils.py @@ -11,10 +11,10 @@ def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: sql = f""" select case + when table_type = 'BASE TABLE' and target_lag is not null then 'dynamic_table' when table_type = 'BASE TABLE' then 'table' when table_type = 'VIEW' then 'view' when table_type = 'EXTERNAL TABLE' then 'external_table' - when table_type is null then 'dynamic_table' end as relation_type from information_schema.tables where table_name like '{relation.identifier.upper()}' From 0773db4c448bc47946f53293c710e2ddea62e369 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 19 Mar 2024 16:41:06 -0500 Subject: [PATCH 3/9] typo --- dbt/include/snowflake/macros/catalog.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index e704771cb..f5737f7a4 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -42,7 +42,7 @@ table_schema as "table_schema", table_name as "table_name", case - when target_lag not null and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + when target_lag is not null and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' else coalesce(table_type, 'BASE TABLE') end as "table_type", comment as "table_comment", From 882b24bb462cb71cd11234c951645748e4413f9c Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 20 Mar 2024 01:06:35 -0500 Subject: [PATCH 4/9] trying to use describe to see target_lag --- dbt/include/snowflake/macros/catalog.sql | 43 ++++++++++--------- .../adapter/dynamic_table_tests/utils.py | 22 ++++++---- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index f5737f7a4..5fd149740 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -38,38 +38,41 @@ {% macro snowflake__get_catalog_tables_sql(information_schema) -%} select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - case - when target_lag is not null and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' - else coalesce(table_type, 'BASE TABLE') - end as "table_type", - comment as "table_comment", - - -- note: this is the _role_ that owns the table - table_owner as "table_owner", + t.table_catalog as "table_database", + t.table_schema as "table_schema", + t.table_name as "table_name", + case + when tp.target_lag is not null and t.table_type = 'BASE TABLE' then 'DYNAMIC TABLE' + else t.table_type + end as "table_type", + t.comment as "table_comment", + t.table_owner as "table_owner", 'Clustering Key' as "stats:clustering_key:label", - clustering_key as "stats:clustering_key:value", + t.clustering_key as "stats:clustering_key:value", 'The key used to cluster this table' as "stats:clustering_key:description", - (clustering_key is not null) as "stats:clustering_key:include", + (t.clustering_key is not null) as "stats:clustering_key:include", 'Row Count' as "stats:row_count:label", - row_count as "stats:row_count:value", + t.row_count as "stats:row_count:value", 'An approximate count of rows in this table' as "stats:row_count:description", - (row_count is not null) as "stats:row_count:include", + (t.row_count is not null) as "stats:row_count:include", 'Approximate Size' as "stats:bytes:label", - bytes as "stats:bytes:value", + t.bytes as "stats:bytes:value", 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", - (bytes is not null) as "stats:bytes:include", + (t.bytes is not null) as "stats:bytes:include", 'Last Modified' as "stats:last_modified:label", - to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", + to_varchar(convert_timezone('UTC', t.last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", 'The timestamp for last update/change' as "stats:last_modified:description", - (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" - from {{ information_schema }}.tables + (t.last_altered is not null and t.table_type = 'BASE TABLE') as "stats:last_modified:include" + from + {{ information_schema }}.tables t + left join + table_properties tp on t.table_catalog = tp.table_catalog + and t.table_schema = tp.table_schema + and t.table_name = tp.table_name; {%- endmacro %} diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py index 87196780c..8ef0477ac 100644 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ b/tests/functional/adapter/dynamic_table_tests/utils.py @@ -11,15 +11,21 @@ def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: sql = f""" select case - when table_type = 'BASE TABLE' and target_lag is not null then 'dynamic_table' - when table_type = 'BASE TABLE' then 'table' - when table_type = 'VIEW' then 'view' - when table_type = 'EXTERNAL TABLE' then 'external_table' + when t.table_type = 'BASE TABLE' and dt.completion_target is not null then 'dynamic_table' + when t.table_type = 'BASE TABLE' then 'table' + when t.table_type = 'VIEW' then 'view' + when t.table_type = 'EXTERNAL TABLE' then 'external_table' end as relation_type - from information_schema.tables - where table_name like '{relation.identifier.upper()}' - and table_schema like '{relation.schema.upper()}' - and table_catalog like '{relation.database.upper()}' + from + information_schema.tables t + left join + information_schema.dynamic_table_refresh_history dt on t.table_catalog = dt.table_catalog + and t.table_schema = dt.schema_name + and t.table_name = dt.name + where + t.table_name like '{relation.identifier.upper()}' + and t.schema_name like '{relation.schema.upper()}' + and t.table_catalog like '{relation.database.upper()}' """ results = project.run_sql(sql, fetch="one") if results is None or len(results) == 0: From a5c636db592514bdb4bae6fd69383a9f73ba92af Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 20 Mar 2024 15:18:33 -0500 Subject: [PATCH 5/9] reverting back and trying to use tables field is_dynamic --- dbt/include/snowflake/macros/catalog.sql | 43 +++++++++---------- .../adapter/dynamic_table_tests/utils.py | 22 ++++------ 2 files changed, 28 insertions(+), 37 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 5fd149740..e434071c3 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -38,41 +38,38 @@ {% macro snowflake__get_catalog_tables_sql(information_schema) -%} select - t.table_catalog as "table_database", - t.table_schema as "table_schema", - t.table_name as "table_name", - case - when tp.target_lag is not null and t.table_type = 'BASE TABLE' then 'DYNAMIC TABLE' - else t.table_type - end as "table_type", - t.comment as "table_comment", - t.table_owner as "table_owner", + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + case + when is_dynamic is not null and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + else table_type + end as "table_type", + comment as "table_comment", + + -- note: this is the _role_ that owns the table + table_owner as "table_owner", 'Clustering Key' as "stats:clustering_key:label", - t.clustering_key as "stats:clustering_key:value", + clustering_key as "stats:clustering_key:value", 'The key used to cluster this table' as "stats:clustering_key:description", - (t.clustering_key is not null) as "stats:clustering_key:include", + (clustering_key is not null) as "stats:clustering_key:include", 'Row Count' as "stats:row_count:label", - t.row_count as "stats:row_count:value", + row_count as "stats:row_count:value", 'An approximate count of rows in this table' as "stats:row_count:description", - (t.row_count is not null) as "stats:row_count:include", + (row_count is not null) as "stats:row_count:include", 'Approximate Size' as "stats:bytes:label", - t.bytes as "stats:bytes:value", + bytes as "stats:bytes:value", 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", - (t.bytes is not null) as "stats:bytes:include", + (bytes is not null) as "stats:bytes:include", 'Last Modified' as "stats:last_modified:label", - to_varchar(convert_timezone('UTC', t.last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", + to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", 'The timestamp for last update/change' as "stats:last_modified:description", - (t.last_altered is not null and t.table_type = 'BASE TABLE') as "stats:last_modified:include" - from - {{ information_schema }}.tables t - left join - table_properties tp on t.table_catalog = tp.table_catalog - and t.table_schema = tp.table_schema - and t.table_name = tp.table_name; + (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" + from {{ information_schema }}.tables {%- endmacro %} diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py index 8ef0477ac..6d79d1792 100644 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ b/tests/functional/adapter/dynamic_table_tests/utils.py @@ -11,21 +11,15 @@ def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: sql = f""" select case - when t.table_type = 'BASE TABLE' and dt.completion_target is not null then 'dynamic_table' - when t.table_type = 'BASE TABLE' then 'table' - when t.table_type = 'VIEW' then 'view' - when t.table_type = 'EXTERNAL TABLE' then 'external_table' + when table_type = 'BASE TABLE' and is_dynamic is not null then 'dynamic_table' + when table_type = 'BASE TABLE' then 'table' + when table_type = 'VIEW' then 'view' + when table_type = 'EXTERNAL TABLE' then 'external_table' end as relation_type - from - information_schema.tables t - left join - information_schema.dynamic_table_refresh_history dt on t.table_catalog = dt.table_catalog - and t.table_schema = dt.schema_name - and t.table_name = dt.name - where - t.table_name like '{relation.identifier.upper()}' - and t.schema_name like '{relation.schema.upper()}' - and t.table_catalog like '{relation.database.upper()}' + from information_schema.tables + where table_name like '{relation.identifier.upper()}' + and table_schema like '{relation.schema.upper()}' + and table_catalog like '{relation.database.upper()}' """ results = project.run_sql(sql, fetch="one") if results is None or len(results) == 0: From f50867d730769bf61a8e048ef5b35a0e56bcc65a Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 20 Mar 2024 15:36:40 -0500 Subject: [PATCH 6/9] changing parameter for is_dynamic validation --- dbt/include/snowflake/macros/catalog.sql | 2 +- tests/functional/adapter/dynamic_table_tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index e434071c3..6a7ddc4b3 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -42,7 +42,7 @@ table_schema as "table_schema", table_name as "table_name", case - when is_dynamic is not null and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + when is_dynamic = true and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' else table_type end as "table_type", comment as "table_comment", diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py index 6d79d1792..bc03ae054 100644 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ b/tests/functional/adapter/dynamic_table_tests/utils.py @@ -11,7 +11,7 @@ def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: sql = f""" select case - when table_type = 'BASE TABLE' and is_dynamic is not null then 'dynamic_table' + when table_type = 'BASE TABLE' and is_dynamic = true then 'dynamic_table' when table_type = 'BASE TABLE' then 'table' when table_type = 'VIEW' then 'view' when table_type = 'EXTERNAL TABLE' then 'external_table' From 963547a58ee0aba37a90cbc02d7b645aa51bb5cb Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 20 Mar 2024 15:41:07 -0500 Subject: [PATCH 7/9] typo, changelog --- .changes/unreleased/Fixes-20240320-154054.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240320-154054.yaml diff --git a/.changes/unreleased/Fixes-20240320-154054.yaml b/.changes/unreleased/Fixes-20240320-154054.yaml new file mode 100644 index 000000000..6d74a2be9 --- /dev/null +++ b/.changes/unreleased/Fixes-20240320-154054.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: modify metadata queries to take into account object type of BASE TABLE being + assigned to dynamic tables when was null previously +time: 2024-03-20T15:40:54.074292-05:00 +custom: + Author: McKnight-42 + Issue: "934" From afb8208ab47871fd28decc2d518686ee3143a8ab Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 20 Mar 2024 15:52:39 -0500 Subject: [PATCH 8/9] update to is_dnyamic expected value --- dbt/include/snowflake/macros/catalog.sql | 2 +- tests/functional/adapter/dynamic_table_tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 6a7ddc4b3..72951a9f0 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -42,7 +42,7 @@ table_schema as "table_schema", table_name as "table_name", case - when is_dynamic = true and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + when is_dynamic = 'yes' and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' else table_type end as "table_type", comment as "table_comment", diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py index bc03ae054..0ff60e2f9 100644 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ b/tests/functional/adapter/dynamic_table_tests/utils.py @@ -11,7 +11,7 @@ def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: sql = f""" select case - when table_type = 'BASE TABLE' and is_dynamic = true then 'dynamic_table' + when table_type = 'BASE TABLE' and is_dynamic = 'yes' then 'dynamic_table' when table_type = 'BASE TABLE' then 'table' when table_type = 'VIEW' then 'view' when table_type = 'EXTERNAL TABLE' then 'external_table' From c50996711b796706a613dc261ab1460438fca1b5 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 20 Mar 2024 16:05:18 -0500 Subject: [PATCH 9/9] take into account uppercase --- dbt/include/snowflake/macros/catalog.sql | 2 +- tests/functional/adapter/dynamic_table_tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 72951a9f0..8354d3c44 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -42,7 +42,7 @@ table_schema as "table_schema", table_name as "table_name", case - when is_dynamic = 'yes' and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' + when is_dynamic = 'YES' and table_type = 'BASE TABLE' THEN 'DYNAMIC TABLE' else table_type end as "table_type", comment as "table_comment", diff --git a/tests/functional/adapter/dynamic_table_tests/utils.py b/tests/functional/adapter/dynamic_table_tests/utils.py index 0ff60e2f9..5763e1b6c 100644 --- a/tests/functional/adapter/dynamic_table_tests/utils.py +++ b/tests/functional/adapter/dynamic_table_tests/utils.py @@ -11,7 +11,7 @@ def query_relation_type(project, relation: SnowflakeRelation) -> Optional[str]: sql = f""" select case - when table_type = 'BASE TABLE' and is_dynamic = 'yes' then 'dynamic_table' + when table_type = 'BASE TABLE' and is_dynamic = 'YES' then 'dynamic_table' when table_type = 'BASE TABLE' then 'table' when table_type = 'VIEW' then 'view' when table_type = 'EXTERNAL TABLE' then 'external_table'