From 161841a128ac67ae0a93e3bc57b4f98fb3d6ebc3 Mon Sep 17 00:00:00 2001 From: Damian Owsianny Date: Wed, 29 Nov 2023 19:46:12 +0100 Subject: [PATCH] Support CREATE OR REPLACE in table materialization --- .../unreleased/Features-20231129-195500.yaml | 7 ++ dbt/include/trino/macros/adapters.sql | 12 +++- .../trino/macros/materializations/table.sql | 8 ++- ..._table_drop.py => test_on_table_exists.py} | 71 ++++++++++++++----- 4 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 .changes/unreleased/Features-20231129-195500.yaml rename tests/functional/adapter/materialization/{test_table_drop.py => test_on_table_exists.py} (52%) diff --git a/.changes/unreleased/Features-20231129-195500.yaml b/.changes/unreleased/Features-20231129-195500.yaml new file mode 100644 index 00000000..cbcf4374 --- /dev/null +++ b/.changes/unreleased/Features-20231129-195500.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Support CREATE OR REPLACE in table materialization +time: 2023-11-29T19:55:00.460963+01:00 +custom: + Author: damian3031 + Issue: "371" + PR: "377" diff --git a/dbt/include/trino/macros/adapters.sql b/dbt/include/trino/macros/adapters.sql index fe7f3a35..faf72b2d 100644 --- a/dbt/include/trino/macros/adapters.sql +++ b/dbt/include/trino/macros/adapters.sql @@ -99,13 +99,19 @@ {%- endif -%} {%- endmacro -%} -{% macro trino__create_table_as(temporary, relation, sql) -%} +{% macro trino__create_table_as(temporary, relation, sql, replace=False) -%} {%- set _properties = config.get('properties') -%} + {%- if replace -%} + {%- set or_replace = ' or replace' -%} + {%- else -%} + {%- set or_replace = '' -%} + {%- endif -%} + {%- set contract_config = config.get('contract') -%} {%- if contract_config.enforced -%} - create table + create{{ or_replace }} table {{ relation }} {{ get_table_columns_and_constraints() }} {{ get_assert_columns_equivalent(sql) }} @@ -122,7 +128,7 @@ {%- else %} - create table {{ relation }} + create{{ or_replace }} table {{ relation }} {{ comment(model.get('description')) }} {{ properties(_properties) }} as ( diff --git a/dbt/include/trino/macros/materializations/table.sql b/dbt/include/trino/macros/materializations/table.sql index bd826306..a082e123 100644 --- a/dbt/include/trino/macros/materializations/table.sql +++ b/dbt/include/trino/macros/materializations/table.sql @@ -1,6 +1,6 @@ {% materialization table, adapter = 'trino' %} {%- set on_table_exists = config.get('on_table_exists', 'rename') -%} - {% if on_table_exists not in ['rename', 'drop'] %} + {% if on_table_exists not in ['rename', 'drop', 'replace'] %} {%- set log_message = 'Invalid value for on_table_exists (%s) specified. Setting default value (%s).' % (on_table_exists, 'rename') -%} {% do log(log_message) %} {%- set on_table_exists = 'rename' -%} @@ -57,6 +57,12 @@ {% call statement('main') -%} {{ create_table_as(False, target_relation, sql) }} {%- endcall %} + + {% elif on_table_exists == 'replace' %} + {#-- build model #} + {% call statement('main') -%} + {{ create_table_as(False, target_relation, sql, True) }} + {%- endcall %} {% endif %} {% do persist_docs(target_relation, model) %} diff --git a/tests/functional/adapter/materialization/test_table_drop.py b/tests/functional/adapter/materialization/test_on_table_exists.py similarity index 52% rename from tests/functional/adapter/materialization/test_table_drop.py rename to tests/functional/adapter/materialization/test_on_table_exists.py index f3830196..36b97246 100644 --- a/tests/functional/adapter/materialization/test_table_drop.py +++ b/tests/functional/adapter/materialization/test_on_table_exists.py @@ -1,5 +1,5 @@ import pytest -from dbt.tests.util import check_relations_equal, run_dbt +from dbt.tests.util import check_relations_equal, run_dbt, run_dbt_and_capture from tests.functional.adapter.materialization.fixtures import ( model_sql, @@ -8,13 +8,29 @@ ) -class TestTableDrop: +class BaseOnTableExists: + # everything that goes in the "seeds" directory + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + # everything that goes in the "models" directory + @pytest.fixture(scope="class") + def models(self): + return { + "materialization.sql": model_sql, + "materialization.yml": profile_yml, + } + + +class TestOnTableExistsDrop(BaseOnTableExists): """ Testing on_table_exists = `drop` configuration for table materialization, using dbt seed, run and tests commands and validate data load correctness. """ - # configuration in dbt_project.yml @pytest.fixture(scope="class") def project_config_update(self): return { @@ -25,19 +41,40 @@ def project_config_update(self): }, } - # everything that goes in the "seeds" directory - @pytest.fixture(scope="class") - def seeds(self): - return { - "seed.csv": seed_csv, - } + # The actual sequence of dbt commands and assertions + # pytest will take care of all "setup" + "teardown" + def test_run_seed_test(self, project): + # seed seeds + results = run_dbt(["seed"], expect_pass=True) + assert len(results) == 1 + # run models two times to check on_table_exists = 'drop' + results = run_dbt(["run"], expect_pass=True) + assert len(results) == 1 + results = run_dbt(["run"], expect_pass=True) + assert len(results) == 1 + # test tests + results = run_dbt(["test"], expect_pass=True) + assert len(results) == 3 + + # check if the data was loaded correctly + check_relations_equal(project.adapter, ["seed", "materialization"]) + + +@pytest.mark.iceberg +class TestOnTableExistsReplace(BaseOnTableExists): + """ + Testing on_table_exists = `replace` configuration for table materialization, + using dbt seed, run and tests commands and validate data load correctness. + """ - # everything that goes in the "models" directory @pytest.fixture(scope="class") - def models(self): + def project_config_update(self): return { - "materialization.sql": model_sql, - "materialization.yml": profile_yml, + "name": "table_drop", + "models": {"+materialized": "table", "+on_table_exists": "replace"}, + "seeds": { + "+column_types": {"some_date": "timestamp(6)"}, + }, } # The actual sequence of dbt commands and assertions @@ -46,11 +83,13 @@ def test_run_seed_test(self, project): # seed seeds results = run_dbt(["seed"], expect_pass=True) assert len(results) == 1 - # run models two times to check on_table_exists = 'drop' - results = run_dbt(["run"], expect_pass=True) + # run models two times to check on_table_exists = 'replace' + results, logs = run_dbt_and_capture(["--debug", "run"], expect_pass=True) assert len(results) == 1 - results = run_dbt(["run"], expect_pass=True) + assert "create or replace table" in logs + results, logs = run_dbt_and_capture(["--debug", "run"], expect_pass=True) assert len(results) == 1 + assert "create or replace table" in logs # test tests results = run_dbt(["test"], expect_pass=True) assert len(results) == 3