Add support for Azure Synapse External Tables

README.md

@@ -121,3 +121,4 @@ as a dbt source and stage-ready external table in Snowflake and Spectrum.
 * Redshift (Spectrum)
 * Snowflake
 * TK: Spark
+* Synapse

macros/external/create_external_table.sql

@@ -93,6 +93,37 @@
     file_format = {{external.file_format}}
 {% endmacro %}
 
+{% macro sqlserver__create_external_table(source_node) %}
+
+    {%- set columns = source_node.columns.values() -%}
+    {%- set external = source_node.external -%}
+    {%- set partitions = external.partitions -%}
+
+    {# TODO add params for modifiying these options? #}
+    SET ANSI_NULLS ON;
+    SET QUOTED_IDENTIFIER ON;
+
+    create external table {{source(source_node.source_name, source_node.name)}} (
+        {% for column in columns %}
+            {# TODO set nullity based on schema tests?? #}
+            {%- set nullity = 'NULL' if 'not_null' in columns.tests else 'NOT NULL'-%}
+            {{adapter.quote(column.name)}} {{column.data_type}} {{nullity}}
+            {{- ',' if not loop.last -}}
+        {% endfor %}
+    )
+    WITH (
+        {% set dict = {'DATA_SOURCE': external.data_source,
+                       'LOCATION' : external.location, 
+                       'FILE_FORMAT' : external.file_format, 
+                       'REJECT_TYPE' : external.reject_type, 
+                       'REJECT_VALUE' : external.reject_value} -%}
+        {%- for key, value in dict.items() %}
+            {{key}} = {% if key == "LOCATION" -%} '{{value}}' {%- else -%} {{value}} {%- endif -%}
+            {{- ',' if not loop.last -}}
+            {%- endfor -%}
+    )
+{% endmacro %}
+
 {% macro bigquery__create_external_table(source_node) %}
     {{ exceptions.raise_compiler_error(
         "BigQuery does not support creating external tables in SQL/DDL.

macros/external/stage_external_sources.sql

@@ -71,6 +71,22 @@
 
 {% endmacro %}
 
+{% macro sqlserver__get_external_build_plan(source_node) %}
+
+    {% set build_plan = [] %}
+
+    {%- set partitions = source_node.external.get('partitions', none) -%}
+    {% set create_or_replace = (var('ext_full_refresh', false)) %}
+
+
+        {% set build_plan = [
+                dbt_external_tables.sqlserver__dropif(source_node),
+                dbt_external_tables.create_external_table(source_node)]%}
+
+    {% do return(build_plan) %}
+
+{% endmacro %}
+
 {% macro stage_external_sources(select=none) %}
 
     {% set sources_to_stage = [] %}

macros/helpers/sqlserver/dropif.sql

@@ -0,0 +1,12 @@
+{% macro sqlserver__dropif(node) %}
+
+    {% set ddl %}
+      if object_id ('{{source(node.source_name, node.name)}}') is not null
+        begin
+        drop external table {{source(node.source_name, node.name)}}
+        end
+    {% endset %}
+
+    {{return(ddl)}}
+
+{% endmacro %}

sample_sources/synapse.yml

@@ -0,0 +1,55 @@
+# Creates query given below
+
+version: 2
+
+sources:
+  - name: marketo
+    schema: source_marketo
+    loader: ADLSblob
+    tables:
+      - name: lead_activities
+        description: |
+          from raw DW.
+        external:
+          data_source: SynapseContainer # External Data Source name (created prior)
+          location: /marketing/Marketo/LeadActivities/ # path on above data source
+          file_format: CommaDelimited # External File Format name (created prior)
+          reject_type: VALUE
+          reject_value: 0
+          columns:
+            - name: id
+              description: unique Activity ID
+              data_type: int
+            - name: leadId
+              description: Lead ID
+              data_type: int
+            - name: activityDate
+              description: date of activity
+              data_type: varchar(255)
+            - name: activityTypeId
+              description: unique identifier for type of activity
+              data_type: int
+            - name: campaignId
+              description: Campaign under which activity took place
+              data_type: int
+            - name: primaryAttributeValueId
+              description: the main attribute for given activity type
+              data_type: int
+            - name: primaryAttributeValue
+              description: what value was taken
+              data_type: varchar(255)
+
+# SET ANSI_NULLS ON;
+# SET QUOTED_IDENTIFIER ON;
+
+# CREATE EXTERNAL TABLE [source].[lead_activities]
+# ( 
+#    [id] [int]  NOT NULL,
+#    [leadId] [int]  NOT NULL,
+#    [activityDate] [varchar](255)  NOT NULL,
+#    [activityTypeId] [int]  NOT NULL,
+#    [campaignId] [int]  NOT NULL,
+#    [primaryAttributeValueId] [int]  NOT NULL,
+#    [primaryAttributeValue] [varchar](255)  NOT NULL
+# )
+# WITH (DATA_SOURCE = [SynapseContainer], LOCATION = N'/marketing/Marketo/LeadActivities/LeadActivities.csv', FILE_FORMAT = [CommaDelimited], REJECT_TYPE = VALUE, REJECT_VALUE = 0 );