diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 768ac3f..3220674 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -4,48 +4,26 @@ **This PR will result in the following new package version:** -**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:** +**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:** + ## PR Checklist ### Basic Validation Please acknowledge that you have successfully performed the following commands locally: -- [ ] dbt compile -- [ ] dbt run –full-refresh -- [ ] dbt run -- [ ] dbt test -- [ ] dbt run –vars (if applicable) +- [ ] dbt run –full-refresh && dbt test +- [ ] dbt run (if incremental models are present) && dbt test Before marking this PR as "ready for review" the following have been applied: -- [ ] The appropriate issue has been linked and tagged -- [ ] You are assigned to the corresponding issue and this PR +- [ ] The appropriate issue has been linked, tagged, and properly assigned +- [ ] All necessary documentation and version upgrades have been applied +- [ ] docs were regenerated (unless this PR does not include any code or yml updates) - [ ] BuildKite integration tests are passing +- [ ] Detailed validation steps have been provided below ### Detailed Validation -Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review": -- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature. -- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages. -- [ ] You have provided details below around the validation steps performed to gain confidence in these changes. +Please share any and all of your validation steps: -### Standard Updates -Please acknowledge that your PR contains the following standard updates: -- Package versioning has been appropriately indexed in the following locations: - - [ ] indexed within dbt_project.yml - - [ ] indexed within integration_tests/dbt_project.yml -- [ ] CHANGELOG has individual entries for each respective change in this PR - -- [ ] README updates have been applied (if applicable) - -- [ ] DECISIONLOG updates have been updated (if applicable) -- [ ] Appropriate yml documentation has been added (if applicable) - -### dbt Docs -Please acknowledge that after the above were all completed the below were applied to your branch: -- [ ] docs were regenerated (unless this PR does not include any code or yml updates) - ### If you had to summarize this PR in an emoji, which would it be? -:dancer: +:dancer: \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 0000000..8ed5853 --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,13 @@ +name: 'auto release' +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + call-workflow-passing-data: + if: github.event.pull_request.merged + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main + secrets: inherit \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 95cfc8f..8086ca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# dbt_zuora v0.3.0 +[PR #12](https://github.com/fivetran/dbt_zuora/pull/12) includes the following breaking changes: + +## Feature Updates +- Introduced the new `zuora__line_item_enhanced` model. This model includes a line item enriched with invoice, subscription, payment, and refund information. This model has been built with the intention of retaining a common line item schema across all other Fivetran billing data models. + +## Under the Hood: +- Updated the pull request templates. +- Included auto-releaser GitHub Actions workflow to automate future releases. + # dbt_zuora v0.2.1 [PR #10](https://github.com/fivetran/dbt_zuora/pull/10) includes the following breaking changes: ## 🔧 Fixes diff --git a/README.md b/README.md index 77cca6c..86f9f7b 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Include the following zuora_source package version in your `packages.yml` file. ```yaml packages: - package: fivetran/zuora - version: [">=0.2.0", "<0.3.0"] # we recommend using ranges to capture non-breaking changes automatically + version: [">=0.3.0", "<0.4.0"] # we recommend using ranges to capture non-breaking changes automatically ``` Do NOT include the `zuora_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well. diff --git a/dbt_project.yml b/dbt_project.yml index 4f7fa60..c433584 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zuora' -version: '0.2.1' +version: '0.3.0' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore index 1d6ff90..a1e2b8f 100644 --- a/integration_tests/.gitignore +++ b/integration_tests/.gitignore @@ -1,4 +1,5 @@ target/ dbt_modules/ logs/ -env/ \ No newline at end of file +env/ +package-lock.yml \ No newline at end of file diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 2bfd8c4..6c73532 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -49,6 +49,6 @@ integration_tests: host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: zuora_integration_tests_1 - threads: 2 + threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2d29657..757f07a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,8 +1,12 @@ name: 'zuora_integration_tests' -version: '0.2.1' +version: '0.3.0' profile: 'integration_tests' config-version: 2 +# For use with validations +models: + +schema: "zuora_{{ var('directed_schema','dev') }}" + vars: zuora_schema: zuora_integration_tests_1 zuora_source: @@ -114,6 +118,7 @@ seeds: zuora_refund_data: +column_types: refund_transaction_time: "timestamp" + refund_date: "timestamp" zuora_subscription_data: +column_types: contract_acceptance_date: "timestamp" diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml index b16d307..4a6b9c1 100644 --- a/integration_tests/packages.yml +++ b/integration_tests/packages.yml @@ -1,2 +1,2 @@ packages: - - local: ../ \ No newline at end of file + - local: ../ diff --git a/integration_tests/tests/consistency/consistency_line_item_enhanced.sql b/integration_tests/tests/consistency/consistency_line_item_enhanced.sql new file mode 100644 index 0000000..2a2f921 --- /dev/null +++ b/integration_tests/tests/consistency/consistency_line_item_enhanced.sql @@ -0,0 +1,31 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +with prod as ( + select * + from {{ target.schema }}_zuora_prod.zuora__line_item_enhanced +), + +dev as ( + select * + from {{ target.schema }}_zuora_dev.zuora__line_item_enhanced +), + +final as ( + -- test will fail if any rows from prod are not found in dev + (select * from prod + except distinct + select * from dev) + + union all -- union since we only care if rows are produced + + -- test will fail if any rows from dev are not found in prod + (select * from dev + except distinct + select * from prod) + ) + +select * +from final \ No newline at end of file diff --git a/integration_tests/tests/consistency/consistency_line_item_enhanced_count.sql b/integration_tests/tests/consistency/consistency_line_item_enhanced_count.sql new file mode 100644 index 0000000..c2b5909 --- /dev/null +++ b/integration_tests/tests/consistency/consistency_line_item_enhanced_count.sql @@ -0,0 +1,21 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- this test is to make sure the rows counts are the same between versions +with prod as ( + select count(*) as prod_rows + from {{ target.schema }}_zuora_prod.zuora__line_item_enhanced +), + +dev as ( + select count(*) as dev_rows + from {{ target.schema }}_zuora_dev.zuora__line_item_enhanced +) + +-- test will return values and fail if the row counts don't match +select * +from prod +join dev + on prod.prod_rows != dev.dev_rows \ No newline at end of file diff --git a/models/common_data_models/docs.md b/models/common_data_models/docs.md new file mode 100644 index 0000000..1d5cd9f --- /dev/null +++ b/models/common_data_models/docs.md @@ -0,0 +1,139 @@ +{% docs billing_type %} +description +{% enddocs %} + +{% docs created_at %} +description +{% enddocs %} + +{% docs currency %} +description +{% enddocs %} + +{% docs customer_city %} +description +{% enddocs %} + +{% docs customer_company %} +description +{% enddocs %} + +{% docs customer_country %} +description +{% enddocs %} + +{% docs customer_email %} +description +{% enddocs %} + +{% docs customer_id %} +description +{% enddocs %} + +{% docs customer_level %} +description +{% enddocs %} + +{% docs customer_name %} +description +{% enddocs %} + +{% docs discount_amount %} +description +{% enddocs %} + +{% docs fee_amount %} +description +{% enddocs %} + +{% docs header_id %} +description +{% enddocs %} + +{% docs header_status %} +description +{% enddocs %} + +{% docs line_item_id %} +description +{% enddocs %} + +{% docs line_item_index %} +description +{% enddocs %} + +{% docs payment_at %} +description +{% enddocs %} + +{% docs payment_id %} +description +{% enddocs %} + +{% docs payment_method %} +description +{% enddocs %} + +{% docs payment_method_id %} +description +{% enddocs %} + +{% docs product_category %} +description +{% enddocs %} + +{% docs product_id %} +description +{% enddocs %} + +{% docs product_name %} +description +{% enddocs %} + +{% docs product_type %} +description +{% enddocs %} + +{% docs quantity %} +description +{% enddocs %} + +{% docs record_type %} +description +{% enddocs %} + +{% docs refund_amount %} +description +{% enddocs %} + +{% docs subscription_id %} +description +{% enddocs %} + +{% docs subscription_period_ended_at %} +description +{% enddocs %} + +{% docs subscription_period_started_at %} +description +{% enddocs %} + +{% docs subscription_status %} +description +{% enddocs %} + +{% docs tax_amount %} +description +{% enddocs %} + +{% docs total_amount %} +description +{% enddocs %} + +{% docs transaction_type %} +description +{% enddocs %} + +{% docs unit_amount %} +description +{% enddocs %} \ No newline at end of file diff --git a/models/common_data_models/zuora__common_data_models.yml b/models/common_data_models/zuora__common_data_models.yml new file mode 100644 index 0000000..dfdca99 --- /dev/null +++ b/models/common_data_models/zuora__common_data_models.yml @@ -0,0 +1,81 @@ +version: 2 + +models: + - name: zuora__line_item_enhanced + description: Add description + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - header_id + - line_item_id + columns: + - name: header_id + description: "{{ doc('header_id') }}" + - name: line_item_id + description: "{{ doc('line_item_id') }}" + - name: line_item_index + description: "{{ doc('line_item_index') }}" + - name: record_type + description: "{{ doc('record_type') }}" + - name: created_at + description: "{{ doc('created_at') }}" + - name: header_status + description: "{{ doc('header_status') }}" + - name: billing_type + description: "{{ doc('billing_type') }}" + - name: currency + description: "{{ doc('currency') }}" + - name: product_id + description: "{{ doc('product_id') }}" + - name: product_name + description: "{{ doc('product_name') }}" + - name: product_type + description: "{{ doc('product_type') }}" + - name: transaction_type + description: "{{ doc('transaction_type') }}" + - name: product_category + description: "{{ doc('product_category') }}" + - name: quantity + description: "{{ doc('quantity') }}" + - name: unit_amount + description: "{{ doc('unit_amount') }}" + - name: discount_amount + description: "{{ doc('discount_amount') }}" + - name: tax_amount + description: "{{ doc('tax_amount') }}" + - name: total_amount + description: "{{ doc('total_amount') }}" + - name: payment_id + description: "{{ doc('payment_id') }}" + - name: payment_method + description: "{{ doc('payment_method') }}" + - name: payment_method_id + description: "{{ doc('payment_method_id') }}" + - name: payment_at + description: "{{ doc('payment_at') }}" + - name: refund_amount + description: "{{ doc('refund_amount') }}" + - name: subscription_id + description: "{{ doc('subscription_id') }}" + - name: subscription_period_started_at + description: "{{ doc('subscription_period_started_at') }}" + - name: subscription_period_ended_at + description: "{{ doc('subscription_period_ended_at') }}" + - name: subscription_status + description: "{{ doc('subscription_status') }}" + - name: customer_id + description: "{{ doc('customer_id') }}" + - name: customer_level + description: "{{ doc('customer_level') }}" + - name: customer_name + description: "{{ doc('customer_name') }}" + - name: customer_company + description: "{{ doc('customer_company') }}" + - name: customer_email + description: "{{ doc('customer_email') }}" + - name: customer_city + description: "{{ doc('customer_city') }}" + - name: customer_country + description: "{{ doc('customer_country') }}" + - name: fee_amount + description: "{{ doc('fee_amount') }}" \ No newline at end of file diff --git a/models/common_data_models/zuora__line_item_enhanced.sql b/models/common_data_models/zuora__line_item_enhanced.sql new file mode 100644 index 0000000..dfde3ca --- /dev/null +++ b/models/common_data_models/zuora__line_item_enhanced.sql @@ -0,0 +1,207 @@ +with line_items as ( + + select * + from {{ var('invoice_item')}} + where is_most_recent_record + +), accounts as ( + + select * + from {{ var('account') }} + where is_most_recent_record + +), contacts as ( + + select * + from {{ var('contact') }} + where is_most_recent_record + +), invoices as ( + + select * + from {{ var('invoice') }} + where is_most_recent_record + +), invoice_payments as ( + + select * + from {{ var('invoice_payment') }} + where is_most_recent_record + +), payments as ( + + select * + from {{ var('payment') }} + where is_most_recent_record + +), payment_methods as ( + + select * + from {{ var('payment_method') }} + where is_most_recent_record + +), products as ( + + select * + from {{ var('product') }} + where is_most_recent_record + +), subscriptions as ( + + select * + from {{ var('subscription') }} + where is_most_recent_record + +), enhanced as ( +select + cast(line_items.invoice_id as {{ dbt.type_string() }}) as header_id, + cast(line_items.invoice_item_id as {{ dbt.type_string() }}) as line_item_id, + cast(row_number() over (partition by line_items.invoice_id order by line_items.invoice_item_id) + as {{ dbt.type_int() }}) as line_item_index, + line_items.created_date as line_item_created_at, + invoices.created_date as invoice_created_at, + invoices.status as header_status, + invoices.source_type as billing_type, + line_items.transaction_currency as currency, + cast(line_items.product_id as {{ dbt.type_string() }}) as product_id, + cast(products.name as {{ dbt.type_string() }}) as product_name, + cast(products.category as {{ dbt.type_string() }}) as product_type, + cast(case + when cast(line_items.processing_type as {{ dbt.type_string() }}) = '0' then 'charge' + when cast(line_items.processing_type as {{ dbt.type_string() }}) = '1' then 'discount' + when cast(line_items.processing_type as {{ dbt.type_string() }}) = '2' then 'prepayment' + when cast(line_items.processing_type as {{ dbt.type_string() }}) = '3' then 'tax' + end as {{ dbt.type_string() }}) as transaction_type, + line_items.quantity, + cast(line_items.unit_price as {{ dbt.type_numeric() }}) as unit_amount, + cast(case when cast(line_items.processing_type as {{ dbt.type_string() }}) = '1' + then line_items.charge_amount else 0 + end as {{ dbt.type_numeric() }}) as discount_amount, + cast(line_items.tax_amount as {{ dbt.type_numeric() }}) as tax_amount, + cast(line_items.charge_amount as {{ dbt.type_numeric() }}) as total_amount, + invoice_payments.payment_id as payment_id, + invoice_payments.payment_method_id, + payment_methods.name as payment_method, + payments.effective_date as payment_at, + cast(invoices.refund_amount as {{ dbt.type_numeric() }}) as refund_amount, + line_items.subscription_id, + subscriptions.subscription_start_date as subscription_period_started_at, + subscriptions.subscription_end_date as subscription_period_ended_at, + subscriptions.status as subscription_status, + line_items.account_id as customer_id, + 'customer' as customer_level, + accounts.name as customer_company, + {{ dbt.concat(["contacts.first_name", "' '", "contacts.last_name"]) }} as customer_name, + contacts.work_email as customer_email, + contacts.city as customer_city, + contacts.country as customer_country + +from line_items + +left join invoices + on invoices.invoice_id = line_items.invoice_id + +left join invoice_payments + on invoice_payments.invoice_id = invoices.invoice_id + +left join payments + on payments.payment_id = invoice_payments.payment_id + +left join payment_methods + on payment_methods.payment_method_id = payments.payment_method_id + +left join accounts + on accounts.account_id = line_items.account_id + +left join contacts + on contacts.contact_id = line_items.bill_to_contact_id + +left join products + on products.product_id = line_items.product_id + +left join subscriptions + on subscriptions.subscription_id = line_items.subscription_id + +), final as ( + + select + header_id, + line_item_id, + line_item_index, + 'line_item' as record_type, + line_item_created_at as created_at, + header_status, + billing_type, + currency, + product_id, + product_name, + product_type, + transaction_type, + quantity, + unit_amount, + discount_amount, + tax_amount, + total_amount, + payment_id, + payment_method_id, + payment_method, + payment_at, + cast(null as {{ dbt.type_numeric() }}) as refund_amount, + subscription_id, + subscription_period_started_at, + subscription_period_ended_at, + subscription_status, + customer_id, + customer_level, + customer_name, + customer_company, + customer_email, + customer_city, + customer_country, + cast(null as {{ dbt.type_numeric() }}) as fee_amount + from enhanced + + union all + + -- Refund information is only reliable at the invoice header. Therefore the below operation creates a new line to track the refund values. + select + header_id, + cast(null as {{ dbt.type_string() }}) as line_item_id, + cast(0 as {{ dbt.type_int() }}) as line_item_index, + 'header' as record_type, + invoice_created_at as created_at, + header_status, + billing_type, + currency, + cast(null as {{ dbt.type_string() }}) as product_id, + cast(null as {{ dbt.type_string() }}) as product_name, + cast(null as {{ dbt.type_string() }}) as product_type, + cast(null as {{ dbt.type_string() }}) as transaction_type, + cast(null as {{ dbt.type_numeric() }}) as quantity, + cast(null as {{ dbt.type_numeric() }}) as unit_amount, + cast(null as {{ dbt.type_numeric() }}) as discount_amount, + cast(null as {{ dbt.type_numeric() }}) as tax_amount, + cast(null as {{ dbt.type_numeric() }}) as total_amount, + payment_id, + payment_method_id, + payment_method, + payment_at, + refund_amount, + cast(null as {{ dbt.type_string() }}) as subscription_id, + cast(null as {{ dbt.type_timestamp() }}) as subscription_period_started_at, + cast(null as {{ dbt.type_timestamp() }}) as subscription_period_ended_at, + cast(null as {{ dbt.type_string() }}) as subscription_status, + customer_id, + customer_level, + customer_name, + customer_company, + customer_email, + customer_city, + customer_country, + cast(null as {{ dbt.type_numeric() }}) as fee_amount + from enhanced + where line_item_index = 1 +) + +select * +from final \ No newline at end of file