From 16e5ff5616b860cd21c5345f9384c3a7fe133005 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Mon, 17 Feb 2025 11:25:52 +0530 Subject: [PATCH 01/12] telemetry --- .../include/global_project/macros/materializations/hooks.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 2e198196c..032f8bdfc 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,4 +1,6 @@ {% macro run_hooks(hooks, inside_transaction=True) %} + {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} + {% set hook_span = tracer.start_span('hooks', context=modules.opentelemetry.context.get_current())%} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} @@ -12,6 +14,7 @@ {% endcall %} {% endif %} {% endfor %} + {% set end_span = hook_span.end() %} {% endmacro %} From 276bb2ec50a114a34f0265628c95490064776c49 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Mon, 17 Feb 2025 11:49:41 +0530 Subject: [PATCH 02/12] refactor --- .../include/global_project/macros/materializations/hooks.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 032f8bdfc..02948bd39 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,6 +1,8 @@ {% macro run_hooks(hooks, inside_transaction=True) %} {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} {% set hook_span = tracer.start_span('hooks', context=modules.opentelemetry.context.get_current())%} + {% set ctx = modules.opentelemetry.trace.set_span_in_context(hook_span) %} + {% set token = modules.opentelemetry.context.attach(ctx) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} @@ -14,7 +16,8 @@ {% endcall %} {% endif %} {% endfor %} - {% set end_span = hook_span.end() %} + {% set _ = modules.opentelemetry.context.detach(token) %} + {% set _ = hook_span.end() %} {% endmacro %} From a82ba2bdfc11e8ea70c6c4557551f7a1bba83ccf Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Mon, 17 Feb 2025 11:57:24 +0530 Subject: [PATCH 03/12] refactor --- .../macros/materializations/hooks.sql | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 02948bd39..3952bacb8 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,23 +1,25 @@ {% macro run_hooks(hooks, inside_transaction=True) %} - {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} - {% set hook_span = tracer.start_span('hooks', context=modules.opentelemetry.context.get_current())%} - {% set ctx = modules.opentelemetry.trace.set_span_in_context(hook_span) %} - {% set token = modules.opentelemetry.context.attach(ctx) %} - {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} - {% if not inside_transaction and loop.first %} - {% call statement(auto_begin=inside_transaction) %} - commit; - {% endcall %} - {% endif %} - {% set rendered = render(hook.get('sql')) | trim %} - {% if (rendered | length) > 0 %} - {% call statement(auto_begin=inside_transaction) %} - {{ rendered }} - {% endcall %} - {% endif %} - {% endfor %} - {% set _ = modules.opentelemetry.context.detach(token) %} - {% set _ = hook_span.end() %} + {% if len(hooks) > 0 %} + {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} + {% set hook_span = tracer.start_span('hooks', context=modules.opentelemetry.context.get_current())%} + {% set ctx = modules.opentelemetry.trace.set_span_in_context(hook_span) %} + {% set token = modules.opentelemetry.context.attach(ctx) %} + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% if not inside_transaction and loop.first %} + {% call statement(auto_begin=inside_transaction) %} + commit; + {% endcall %} + {% endif %} + {% set rendered = render(hook.get('sql')) | trim %} + {% if (rendered | length) > 0 %} + {% call statement(auto_begin=inside_transaction) %} + {{ rendered }} + {% endcall %} + {% endif %} + {% endfor %} + {% set _ = modules.opentelemetry.context.detach(token) %} + {% set _ = hook_span.end() %} + {% endif %} {% endmacro %} From ec926a0136a5c654e8f2c10ddbb7ff774b9281e0 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Mon, 17 Feb 2025 12:19:51 +0530 Subject: [PATCH 04/12] refactor --- .../global_project/macros/materializations/hooks.sql | 8 ++++---- .../include/snowflake/macros/relations/view/create.sql | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 3952bacb8..8bf8d9e2c 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,14 +1,14 @@ -{% macro run_hooks(hooks, inside_transaction=True) %} - {% if len(hooks) > 0 %} +{% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} + {% if hooks|length > 0 %} {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} - {% set hook_span = tracer.start_span('hooks', context=modules.opentelemetry.context.get_current())%} + {% set hook_span = tracer.start_span(span_name, context=modules.opentelemetry.context.get_current())%} {% set ctx = modules.opentelemetry.trace.set_span_in_context(hook_span) %} {% set token = modules.opentelemetry.context.attach(ctx) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} commit; - {% endcall %} +{% endcall %} {% endif %} {% set rendered = render(hook.get('sql')) | trim %} {% if (rendered | length) > 0 %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql index d518ac3e1..0a498d2c6 100644 --- a/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/relations/view/create.sql @@ -49,7 +49,7 @@ eventually be retired in favor of a standardized approach. Changed line: type='view') -%} {% set grant_config = config.get('grants') %} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks, span_name='pre-hook') }} -- If there's a table with the same name and we weren't told to full refresh, -- that's an error. If we were told to full refresh, drop it. This behavior differs @@ -66,7 +66,7 @@ eventually be retired in favor of a standardized approach. Changed line: {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks, span_name='post-hook') }} {{ return({'relations': [target_relation]}) }} From 4063808a97b6ed387384baad9582c9d3e0a143c7 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Tue, 18 Feb 2025 17:13:22 +0530 Subject: [PATCH 05/12] refactor --- .../include/global_project/macros/materializations/hooks.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 8bf8d9e2c..69959c762 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,7 +1,7 @@ {% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} {% if hooks|length > 0 %} {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} - {% set hook_span = tracer.start_span(span_name, context=modules.opentelemetry.context.get_current())%} + {% set hook_span = tracer.start_span(span_name) %} {% set ctx = modules.opentelemetry.trace.set_span_in_context(hook_span) %} {% set token = modules.opentelemetry.context.attach(ctx) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} From 43b6bcb209d15a479fd940390a21e3f8943ad7a2 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Wed, 26 Feb 2025 20:15:43 +0530 Subject: [PATCH 06/12] try with block --- .../global_project/macros/materializations/hooks.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 69959c762..3d0ce8b9e 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,9 +1,7 @@ {% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} {% if hooks|length > 0 %} - {% set tracer = modules.opentelemetry.trace.get_tracer("dbt-runner") %} - {% set hook_span = tracer.start_span(span_name) %} - {% set ctx = modules.opentelemetry.trace.set_span_in_context(hook_span) %} - {% set token = modules.opentelemetry.context.attach(ctx) %} + {% set tracer = modules.opentelemetry.trace.get_tracer("dbt.runner") %} + {% with tracer.start_as_current_span(span_name) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} @@ -17,8 +15,7 @@ {% endcall %} {% endif %} {% endfor %} - {% set _ = modules.opentelemetry.context.detach(token) %} - {% set _ = hook_span.end() %} + {% endwith %} {% endif %} {% endmacro %} From 0199195eae851961c7337b9f8757af9879c24a74 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Wed, 26 Feb 2025 20:27:17 +0530 Subject: [PATCH 07/12] refactor --- .../include/global_project/macros/materializations/hooks.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 3d0ce8b9e..49f95fb06 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,7 +1,7 @@ {% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} {% if hooks|length > 0 %} {% set tracer = modules.opentelemetry.trace.get_tracer("dbt.runner") %} - {% with tracer.start_as_current_span(span_name) %} + {% with _ = tracer.start_as_current_span(span_name) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} From e084d3b99d2907bd6a8721f9c9bed17c18315aba Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Wed, 26 Feb 2025 20:54:59 +0530 Subject: [PATCH 08/12] refactor --- .../include/global_project/macros/materializations/hooks.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 49f95fb06..1aa36a77a 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,7 +1,8 @@ {% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} {% if hooks|length > 0 %} {% set tracer = modules.opentelemetry.trace.get_tracer("dbt.runner") %} - {% with _ = tracer.start_as_current_span(span_name) %} + {% set hook_span = tracer.start_span(span_name) %} + {% with _ = modules.opentelemetry.trace.use_span(hook_span, end_on_exit=True) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} From 702dd30fbedc23c0f512f3a4456b31c58f411e8a Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Wed, 26 Feb 2025 21:32:30 +0530 Subject: [PATCH 09/12] refactor --- .../include/global_project/macros/materializations/hooks.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 1aa36a77a..24bf60c33 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,8 +1,5 @@ {% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} {% if hooks|length > 0 %} - {% set tracer = modules.opentelemetry.trace.get_tracer("dbt.runner") %} - {% set hook_span = tracer.start_span(span_name) %} - {% with _ = modules.opentelemetry.trace.use_span(hook_span, end_on_exit=True) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} @@ -16,7 +13,6 @@ {% endcall %} {% endif %} {% endfor %} - {% endwith %} {% endif %} {% endmacro %} From 3b78c9d851a668cd8bef0af2a6d4989424182a47 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Thu, 27 Feb 2025 00:36:30 +0530 Subject: [PATCH 10/12] refactor --- .../macros/materializations/hooks.sql | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index 24bf60c33..a7ec67acc 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -1,19 +1,17 @@ {% macro run_hooks(hooks, inside_transaction=True, span_name='run_hooks') %} - {% if hooks|length > 0 %} - {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} - {% if not inside_transaction and loop.first %} - {% call statement(auto_begin=inside_transaction) %} - commit; + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% if not inside_transaction and loop.first %} + {% call statement(auto_begin=inside_transaction) %} + commit; {% endcall %} - {% endif %} - {% set rendered = render(hook.get('sql')) | trim %} - {% if (rendered | length) > 0 %} - {% call statement(auto_begin=inside_transaction) %} - {{ rendered }} - {% endcall %} - {% endif %} - {% endfor %} - {% endif %} + {% endif %} + {% set rendered = render(hook.get('sql')) | trim %} + {% if (rendered | length) > 0 %} + {% call statement(auto_begin=inside_transaction) %} + {{ rendered }} + {% endcall %} + {% endif %} + {% endfor %} {% endmacro %} From 1ba21b48b5e2ed51cc7856c23d8dc5a01c058279 Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Thu, 27 Feb 2025 00:37:11 +0530 Subject: [PATCH 11/12] refactor --- .../include/global_project/macros/materializations/hooks.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql index a7ec67acc..09006a8a6 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/hooks.sql @@ -3,7 +3,7 @@ {% if not inside_transaction and loop.first %} {% call statement(auto_begin=inside_transaction) %} commit; -{% endcall %} + {% endcall %} {% endif %} {% set rendered = render(hook.get('sql')) | trim %} {% if (rendered | length) > 0 %} From 8a3591121ccd0cf1e8de1de2e0ab99f30b2acdee Mon Sep 17 00:00:00 2001 From: Vengal Rao Guttha Date: Thu, 27 Feb 2025 00:40:28 +0530 Subject: [PATCH 12/12] name hook spans --- .../snowflake/macros/materializations/dynamic_table.sql | 4 ++-- .../include/snowflake/macros/materializations/incremental.sql | 4 ++-- .../dbt/include/snowflake/macros/materializations/table.sql | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql index f491ef3bd..eb77ba82d 100644 --- a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/dynamic_table.sql @@ -5,7 +5,7 @@ {% set existing_relation = load_cached_relation(this) %} {% set target_relation = this.incorporate(type=this.DynamicTable) %} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks, span_name='pre-hook') }} {% set build_sql = dynamic_table_get_build_sql(existing_relation, target_relation) %} @@ -15,7 +15,7 @@ {{ dynamic_table_execute_build_sql(build_sql, existing_relation, target_relation) }} {% endif %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks, span_name='post-hook') }} {% do unset_query_tag(query_tag) %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql index dbb79de02..71adf4385 100644 --- a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/incremental.sql @@ -86,7 +86,7 @@ {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks, span_name='pre-hook') }} {% if existing_relation is none %} {%- call statement('main', language=language) -%} @@ -148,7 +148,7 @@ {% do drop_relation_if_exists(tmp_relation) %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks, span_name='post-hook') }} {% set target_relation = target_relation.incorporate(type='table') %} diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql index 995757b6b..1e906d705 100644 --- a/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/materializations/table.sql @@ -16,7 +16,7 @@ table_format=config.get('table_format', 'default') ) -%} - {{ run_hooks(pre_hooks) }} + {{ run_hooks(pre_hooks, span_name='pre-hook') }} {% if target_relation.needs_to_drop(existing_relation) %} {{ drop_relation_if_exists(existing_relation) }} @@ -26,7 +26,7 @@ {{ create_table_as(False, target_relation, compiled_code, language) }} {%- endcall %} - {{ run_hooks(post_hooks) }} + {{ run_hooks(post_hooks, span_name='post-hook') }} {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}