From 94af50e683c7c4466708e40e29ce38a51ee102b8 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Mon, 13 May 2024 17:24:38 -0600 Subject: [PATCH 1/2] Cross-database `date` macro (#1030) * Cross-database `date` macro * Temporary dev requirements for testing * Update changelog entry * Revert "Temporary dev requirements for testing" This reverts commit 529461f475c5a639a6d545ff61ab32d91616fd2a. --- .changes/unreleased/Features-20240501-151904.yaml | 6 ++++++ dbt/include/spark/macros/utils/date.sql | 5 +++++ tests/functional/adapter/utils/test_utils.py | 5 +++++ 3 files changed, 16 insertions(+) create mode 100644 .changes/unreleased/Features-20240501-151904.yaml create mode 100644 dbt/include/spark/macros/utils/date.sql diff --git a/.changes/unreleased/Features-20240501-151904.yaml b/.changes/unreleased/Features-20240501-151904.yaml new file mode 100644 index 000000000..ed08f1251 --- /dev/null +++ b/.changes/unreleased/Features-20240501-151904.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Cross-database `date` macro +time: 2024-05-01T15:19:04.822157-06:00 +custom: + Author: dbeatty10 + Issue: 1031 diff --git a/dbt/include/spark/macros/utils/date.sql b/dbt/include/spark/macros/utils/date.sql new file mode 100644 index 000000000..a76e995b3 --- /dev/null +++ b/dbt/include/spark/macros/utils/date.sql @@ -0,0 +1,5 @@ +{% macro spark__date(year, month, day) -%} + {%- set dt = modules.datetime.date(year, month, day) -%} + {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%} + to_date('{{ iso_8601_formatted_date }}', 'yyyy-MM-dd') +{%- endmacro %} diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 0dc526564..e8ebb087f 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -8,6 +8,7 @@ from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_concat import BaseConcat from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive +from dbt.tests.adapter.utils.test_date import BaseDate from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc @@ -80,6 +81,10 @@ class TestCurrentTimestamp(BaseCurrentTimestampNaive): pass +class TestDate(BaseDate): + pass + + class TestDateAdd(BaseDateAdd): pass From 629ea94098cea5f4c182b123048086cd2a48c767 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 14 May 2024 12:28:51 -0400 Subject: [PATCH 2/2] [Bug] Fix incremental python tables - dbt can't find temporary table transaction logs (#1042) * unskip test * align test pyodbc version with build deps; add pyspark from build deps * revert change to exclude database and schema from temp relations * retain existing behavior for unit tests by vendoring temp relation into the incremental materialization * changelog entry --- .changes/unreleased/Fixes-20240513-160121.yaml | 7 +++++++ .../macros/materializations/incremental/incremental.sql | 2 +- requirements.txt | 3 ++- tests/functional/adapter/test_python_model.py | 7 ------- 4 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240513-160121.yaml diff --git a/.changes/unreleased/Fixes-20240513-160121.yaml b/.changes/unreleased/Fixes-20240513-160121.yaml new file mode 100644 index 000000000..fd7072954 --- /dev/null +++ b/.changes/unreleased/Fixes-20240513-160121.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix incremental python models error where Databricks could not find the temp + table transaction logs +time: 2024-05-13T16:01:21.255833-04:00 +custom: + Author: mikealfare + Issue: "1033" diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 10d4f3ed8..9a66bab51 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -16,7 +16,7 @@ {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%} {%- set target_relation = this -%} {%- set existing_relation = load_relation(this) -%} - {%- set tmp_relation = make_temp_relation(this) -%} + {% set tmp_relation = this.incorporate(path = {"identifier": this.identifier ~ '__dbt_tmp'}) -%} {#-- for SQL model we will create temp view that doesn't have database and schema --#} {%- if language == 'sql'-%} diff --git a/requirements.txt b/requirements.txt index 40335fb8a..b32884c43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ pyhive[hive_pure_sasl]~=0.7.0 requests>=2.28.1 -pyodbc~=5.0.1 --no-binary pyodbc +pyodbc~=4.0.39 --no-binary pyodbc sqlparams>=3.0.0 thrift>=0.13.0 +pyspark>=3.0.0,<4.0.0 sqlparse>=0.4.2 # not directly required, pinned by Snyk to avoid a vulnerability types-PyYAML diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 05e25c5f4..1195cbd3e 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -24,13 +24,6 @@ class TestPythonIncrementalModelSpark(BasePythonIncrementalTests): def project_config_update(self): return {} - @pytest.mark.skip( - "Databricks can't find the transaction log" - "See https://github.com/dbt-labs/dbt-spark/issues/1033" - ) - def test_incremental(self, project): - super().test_incremental(project) - models__simple_python_model = """ import pandas