diff --git a/.circleci/config.yml b/.circleci/config.yml index 8869c8bb..2432273b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: integration-postgres: docker: - - image: circleci/python:3.6.13-stretch + - image: cimg/python:3.9.9 - image: circleci/postgres:9.6.5-alpine-ram steps: @@ -24,7 +24,7 @@ jobs: integration-redshift: docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: @@ -35,7 +35,7 @@ jobs: integration-snowflake: docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: @@ -48,7 +48,7 @@ jobs: environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" docker: - - image: circleci/python:3.6.3-stretch + - image: cimg/python:3.9.9 steps: - checkout - run: diff --git a/CHANGELOG.md b/CHANGELOG.md index 139b9c99..6ad5e3e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# dbt-utils v0.8.0 +## 🚨 Breaking changes +- dbt ONE POINT OH is here! This version of dbt-utils requires _any_ version (minor and patch) of v1, which means far less need for compatibility releases in the future. +- The partition column in the `mutually_exclusive_ranges` test is now always called `partition_by_col`. This enables compatibility with `--store-failures` when multiple columns are concatenated together. If you have models built on top of the failures table, update them to reflect the new column name. ([#423](https://github.com/dbt-labs/dbt-utils/issues/423), [#430](https://github.com/dbt-labs/dbt-utils/pull/430)) + +## Contributors: +- [codigo-ergo-sum](https://github.com/codigo-ergo-sum) (#430) + # dbt-utils 0.7.5 🚨 This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). Projects using dbt-utils 0.7.4 with dbt-core v1.0.0 can expect to see a deprecation warning. This will be resolved in dbt_utils v0.8.0. @@ -28,10 +36,12 @@ - Bump `require-dbt-version` to have an upper bound of `'<=1.0.0'`. - Url link fixes within the README for `not_constant`, `dateadd`, `datediff` and updated the header `Logger` to `Jinja Helpers`. ([#431](https://github.com/dbt-labs/dbt-utils/pull/431)) - Fully qualified a `cte_name.*` in the `equality` test to avoid an Exasol error ([#420](https://github.com/dbt-labs/dbt-utils/pull/420)) +- `get_url_host()` macro now correctly handles URLs beginning with `android-app://` ([#426](https://github.com/dbt-labs/dbt-utils/pull/426)) ## Contributors: - [joemarkiewicz](https://github.com/fivetran-joemarkiewicz) - [TimoKruth](https://github.com/TimoKruth) +- [foundinblank](https://github.com/foundinblank) # dbt-utils v0.7.3 @@ -112,6 +122,7 @@ If you were relying on the position to match up your optional arguments, this ma ## Under the hood * Update the default implementation of concat macro to use `||` operator ([#373](https://github.com/fishtown-analytics/dbt-utils/pull/314) from [@ChristopheDuong](https://github.com/ChristopheDuong)). Note this may be a breaking change for adapters that support `concat()` but not `||`, such as Apache Spark. - Use `power()` instead of `pow()` in `generate_series()` and `haversine_distance()` as they are synonyms in most SQL dialects, but some dialects only have `power()` ([#354](https://github.com/fishtown-analytics/dbt-utils/pull/354) from [@swanderz](https://github.com/swanderz)) +- Make `get_column_values` return the default value passed as a parameter instead of an empty string before compilation ([#304](https://github.com/dbt-labs/dbt-utils/pull/386) from [@jmriego](https://github.com/jmriego) # dbt-utils v0.6.6 diff --git a/README.md b/README.md index 72861e16..ca46d193 100644 --- a/README.md +++ b/README.md @@ -726,7 +726,7 @@ group by 1,2,3 ``` #### star ([source](macros/sql/star.sql)) -This macro generates a list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with the star macro. This macro also has an optional `relation_alias` argument that will prefix all generated fields with an alias. +This macro generates a list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with the star macro. This macro also has an optional `relation_alias` argument that will prefix all generated fields with an alias (`relation_alias`.`field_name`). The macro also has optional `prefix` and `suffix` arguments, which will be appropriately concatenated to each field name in the output (`prefix` ~ `field_name` ~ `suffix`). **Usage:** ```sql diff --git a/dbt_project.yml b/dbt_project.yml index a0330712..833501d8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,11 +1,11 @@ name: 'dbt_utils' -version: '0.7.0' +version: '0.1.0' -require-dbt-version: [">=0.20.0", "<1.1.0"] +require-dbt-version: [">=1.0.0", "<2.0.0"] config-version: 2 target-path: "target" -clean-targets: ["target", "dbt_modules"] +clean-targets: ["target", "dbt_modules", "dbt_packages"] macro-paths: ["macros"] log-path: "logs" diff --git a/integration_tests/data/sql/data_star_prefix_suffix_expected.csv b/integration_tests/data/sql/data_star_prefix_suffix_expected.csv new file mode 100644 index 00000000..fd34199b --- /dev/null +++ b/integration_tests/data/sql/data_star_prefix_suffix_expected.csv @@ -0,0 +1,4 @@ +prefix_field_1_suffix,prefix_field_2_suffix,prefix_field_3_suffix +a,b,c +d,e,f +g,h,i diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 9631a8c4..bb6d147b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,16 +8,17 @@ profile: 'integration_tests' config-version: 2 -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_modules" + - "dbt_packages" dispatch: - macro_namespace: 'dbt_utils' @@ -56,9 +57,8 @@ seeds: +schema: events data_get_column_values_dropped: - # TODO: Ideally this would use the adapter, but it gives a "Tried to drop relation, but its type is null" error. - #+post-hook: "{% do adapter.drop_relation(this) %}" - +post-hook: "drop table {{ this }}" + # this.incorporate() to hardcode the node's type as otherwise dbt doesn't know it yet + +post-hook: "{% do adapter.drop_relation(this.incorporate(type='table')) %}" schema_tests: diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index 0e8b2497..5b341457 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -106,6 +106,11 @@ models: - dbt_utils.equality: compare_model: ref('data_star_expected') + - name: test_star_prefix_suffix + tests: + - dbt_utils.equality: + compare_model: ref('data_star_prefix_suffix_expected') + - name: test_surrogate_key tests: - assert_equal: diff --git a/integration_tests/models/sql/test_star_prefix_suffix.sql b/integration_tests/models/sql/test_star_prefix_suffix.sql new file mode 100644 index 00000000..ae1c64fe --- /dev/null +++ b/integration_tests/models/sql/test_star_prefix_suffix.sql @@ -0,0 +1,13 @@ +{% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} +{% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} + +with data as ( + + select + {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} + + from {{ ref('data_star') }} + +) + +select * from data \ No newline at end of file diff --git a/macros/schema_tests/mutually_exclusive_ranges.sql b/macros/schema_tests/mutually_exclusive_ranges.sql index f2da3b95..f88efbe4 100644 --- a/macros/schema_tests/mutually_exclusive_ranges.sql +++ b/macros/schema_tests/mutually_exclusive_ranges.sql @@ -35,7 +35,7 @@ with window_functions as ( select {% if partition_by %} - {{ partition_by }}, + {{ partition_by }} as partition_by_col, {% endif %} {{ lower_bound_column }} as lower_bound, {{ upper_bound_column }} as upper_bound, diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 61b6bece..57b150a6 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -3,10 +3,12 @@ {% endmacro %} {% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - +{% if default is none %} + {% set default = [] %} +{% endif %} {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} - {{ return('') }} + {{ return(default) }} {% endif %} {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index d659e7cb..ffe48946 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,8 +1,8 @@ -{% macro star(from, relation_alias=False, except=[]) -%} - {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except)) }} +{% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%} + {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} {% endmacro %} -{% macro default__star(from, relation_alias=False, except=[]) -%} +{% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%} {%- do dbt_utils._is_relation(from, 'star') -%} {%- do dbt_utils._is_ephemeral(from, 'star') -%} @@ -24,7 +24,7 @@ {%- for col in include_cols %} - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} + {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- if not loop.last %},{{ '\n ' }}{% endif %} {%- endfor -%} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index d6662576..43a58cc4 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -24,4 +24,4 @@ dbt_utils.type_string() )}} -{%- endmacro %} \ No newline at end of file +{%- endmacro %}