Merge branch 'duckdb:master' into feature/refactor-external-materiali…

…zation

.github/workflows/release.yml

@@ -0,0 +1,97 @@
+name: Publish Python 🐍 distribution 📦 to PyPI
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/dbt-duckdb
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign the Python 🐍 distribution 📦 with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/[email protected]
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        '${{ github.ref_name }}'
+        --repo '${{ github.repository }}'
+        --title '${{ github.ref_name }}'
+        --generate-notes
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dist/**
+        --repo '${{ github.repository }}'

dbt/adapters/duckdb/__version__.py

@@ -1 +1 @@
-version = "1.7.2"
+version = "1.7.3"

dbt/adapters/duckdb/connections.py

@@ -3,8 +3,7 @@
 from contextlib import contextmanager
 from typing import Optional
 from typing import Tuple
-
-import agate
+from typing import TYPE_CHECKING
 
 import dbt.exceptions
 from . import environments
@@ -15,6 +14,9 @@
 from dbt.contracts.connection import ConnectionState
 from dbt.logger import GLOBAL_LOGGER as logger
 
+if TYPE_CHECKING:
+    import agate
+
 
 class DuckDBConnectionManager(SQLConnectionManager):
     TYPE = "duckdb"
@@ -101,7 +103,7 @@ def execute(
         auto_begin: bool = False,
         fetch: bool = False,
         limit: Optional[int] = None,
-    ) -> Tuple[AdapterResponse, agate.Table]:
+    ) -> Tuple[AdapterResponse, "agate.Table"]:
         if self.disable_transactions:
             auto_begin = False
         return super().execute(sql, auto_begin, fetch, limit)

dbt/adapters/duckdb/credentials.py

@@ -146,6 +146,15 @@ class DuckDBCredentials(Credentials):
     # by networking issues)
     retries: Optional[Retries] = None
 
+    def __post_init__(self):
+        # Add MotherDuck plugin if the path is a MotherDuck database
+        # and plugin was not specified in profile.yml
+        if self.is_motherduck:
+            if self.plugins is None:
+                self.plugins = []
+            if "motherduck" not in [plugin.module for plugin in self.plugins]:
+                self.plugins.append(PluginConfig(module="motherduck"))
+
     @property
     def is_motherduck(self):
         parsed = urlparse(self.path)

dbt/adapters/duckdb/environments/__init__.py

@@ -87,6 +87,12 @@ def __init__(self, creds: DuckDBCredentials):
                 if path not in sys.path:
                     sys.path.append(path)
 
+        major, minor, patch = [int(x) for x in duckdb.__version__.split(".")]
+        if major == 0 and (minor < 10 or (minor == 10 and patch == 0)):
+            self._supports_comments = False
+        else:
+            self._supports_comments = True
+
     @property
     def creds(self) -> DuckDBCredentials:
         return self._creds
@@ -112,6 +118,9 @@ def store_relation(
     def get_binding_char(self) -> str:
         return "?"
 
+    def supports_comments(self) -> bool:
+        return self._supports_comments
+
     @classmethod
     def initialize_db(
         cls, creds: DuckDBCredentials, plugins: Optional[Dict[str, BasePlugin]] = None

dbt/adapters/duckdb/impl.py

@@ -3,8 +3,7 @@
 from typing import List
 from typing import Optional
 from typing import Sequence
-
-import agate
+from typing import TYPE_CHECKING
 
 from dbt.adapters.base import BaseRelation
 from dbt.adapters.base.column import Column as BaseColumn
@@ -28,6 +27,9 @@
 TEMP_SCHEMA_NAME = "temp_schema_name"
 DEFAULT_TEMP_SCHEMA_NAME = "dbt_temp"
 
+if TYPE_CHECKING:
+    import agate
+
 
 class DuckDBAdapter(SQLAdapter):
     ConnectionManager = DuckDBConnectionManager
@@ -61,7 +63,9 @@ def is_motherduck(self):
         return self.config.credentials.is_motherduck
 
     @available
-    def convert_datetimes_to_strs(self, table: agate.Table) -> agate.Table:
+    def convert_datetimes_to_strs(self, table: "agate.Table") -> "agate.Table":
+        import agate
+
         for column in table.columns:
             if isinstance(column.data_type, agate.DateTime):
                 table = table.compute(
@@ -118,6 +122,13 @@ def external_root(self) -> str:
     def get_binding_char(self):
         return DuckDBConnectionManager.env().get_binding_char()
 
+    @available
+    def catalog_comment(self, prefix):
+        if DuckDBConnectionManager.env().supports_comments():
+            return f"{prefix}.comment"
+        else:
+            return "''"
+
     @available
     def external_write_options(self, write_location: str, rendered_options: dict) -> str:
         if "format" not in rendered_options:

dbt/adapters/duckdb/plugins/motherduck.py

@@ -4,6 +4,7 @@
 from duckdb import DuckDBPyConnection
 
 from . import BasePlugin
+from dbt.adapters.duckdb.__version__ import version as __plugin_version__
 from dbt.adapters.duckdb.credentials import DuckDBCredentials
 from dbt.version import __version__
 
@@ -30,7 +31,7 @@ def token_from_config(creds: DuckDBCredentials) -> str:
         return ""
 
     def update_connection_config(self, creds: DuckDBCredentials, config: Dict[str, Any]):
-        user_agent = f"dbt/{__version__}"
+        user_agent = f"dbt/{__version__} dbt-duckdb/{__plugin_version__}"
         if "custom_user_agent" in config:
             user_agent = f"{user_agent} {config['custom_user_agent']}"
         settings: Dict[str, Any] = creds.settings or {}

dbt/include/duckdb/macros/catalog.sql

@@ -1,28 +1,45 @@
 
 {% macro duckdb__get_catalog(information_schema, schemas) -%}
   {%- call statement('catalog', fetch_result=True) -%}
+    with relations AS (
+      select
+        t.table_name
+        , t.database_name
+        , t.schema_name
+        , 'BASE TABLE' as table_type
+        , {{ adapter.catalog_comment('t') }} as table_comment
+      from duckdb_tables() t
+      WHERE t.database_name = '{{ database }}'
+      UNION ALL
+      SELECT v.view_name as table_name
+      , v.database_name
+      , v.schema_name
+      , 'VIEW' as table_type
+      , {{ adapter.catalog_comment('v') }} as table_comment
+      from duckdb_views() v
+      WHERE v.database_name = '{{ database }}'
+    )
     select
         '{{ database }}' as table_database,
-        t.table_schema,
-        t.table_name,
-        t.table_type,
-        '' as table_comment,
+        r.schema_name as table_schema,
+        r.table_name,
+        r.table_type,
+        r.table_comment,
         c.column_name,
-        c.ordinal_position as column_index,
-        c.data_type column_type,
-        '' as column_comment,
+        c.column_index as column_index,
+        c.data_type as column_type,
+        {{ adapter.catalog_comment('c') }} as column_comment,
         '' as table_owner
-    FROM information_schema.tables t JOIN information_schema.columns c ON t.table_schema = c.table_schema AND t.table_name = c.table_name
+    FROM relations r JOIN duckdb_columns() c ON r.schema_name = c.schema_name AND r.table_name = c.table_name
     WHERE (
         {%- for schema in schemas -%}
-          upper(t.table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
+          upper(r.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
         {%- endfor -%}
     )
-    AND t.table_type IN ('BASE TABLE', 'VIEW')
     ORDER BY
-        t.table_schema,
-        t.table_name,
-        c.ordinal_position
+        r.schema_name,
+        r.table_name,
+        c.column_index
   {%- endcall -%}
   {{ return(load_result('catalog').table) }}
 {%- endmacro %}

dbt/include/duckdb/macros/persist_docs.sql

@@ -0,0 +1,36 @@
+
+{#
+  The logic in this file is adapted from dbt-postgres, since DuckDB matches
+  the Postgres relation/column commenting model as of 0.10.1
+#}
+
+{#
+  By using dollar-quoting like this, users can embed anything they want into their comments
+  (including nested dollar-quoting), as long as they do not use this exact dollar-quoting
+  label. It would be nice to just pick a new one but eventually you do have to give up.
+#}
+{% macro duckdb_escape_comment(comment) -%}
+  {% if comment is not string %}
+    {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}
+  {% endif %}
+  {%- set magic = '$dbt_comment_literal_block$' -%}
+  {%- if magic in comment -%}
+    {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}
+  {%- endif -%}
+  {{ magic }}{{ comment }}{{ magic }}
+{%- endmacro %}
+
+{% macro duckdb__alter_relation_comment(relation, comment) %}
+  {% set escaped_comment = duckdb_escape_comment(comment) %}
+  comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};
+{% endmacro %}
+
+
+{% macro duckdb__alter_column_comment(relation, column_dict) %}
+  {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %}
+  {% for column_name in column_dict if (column_name in existing_columns) %}
+    {% set comment = column_dict[column_name]['description'] %}
+    {% set escaped_comment = duckdb_escape_comment(comment) %}
+    comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};
+  {% endfor %}
+{% endmacro %}

dev-requirements.txt

@@ -2,7 +2,7 @@
 # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
 # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
 
-dbt-tests-adapter==1.7.8
+dbt-tests-adapter==1.7.10
 
 boto3
 mypy-boto3-glue
@@ -15,7 +15,7 @@ freezegun==1.4.0
 fsspec
 gspread
 ipdb
-mypy==1.8.0
+mypy==1.9.0
 openpyxl
 pip-tools
 pre-commit

tests/functional/adapter/test_persist_docs.py

@@ -0,0 +1,21 @@
+import pytest
+
+from dbt.tests.adapter.persist_docs.test_persist_docs import (
+    BasePersistDocs,
+    BasePersistDocsColumnMissing,
+    BasePersistDocsCommentOnQuotedColumn,
+)
+
+@pytest.mark.skip_profile("md")
+class TestPersistDocs(BasePersistDocs):
+    pass
+
+
+@pytest.mark.skip_profile("md")
+class TestPersistDocsColumnMissing(BasePersistDocsColumnMissing):
+    pass
+
+
+@pytest.mark.skip_profile("md")
+class TestPersistDocsCommentOnQuotedColumn(BasePersistDocsCommentOnQuotedColumn):
+    pass

tests/functional/plugins/test_motherduck.py

@@ -8,6 +8,7 @@
 from dbt.adapters.duckdb.credentials import DuckDBCredentials
 from dbt.adapters.duckdb.credentials import PluginConfig
 from dbt.adapters.duckdb.plugins.motherduck import Plugin
+from dbt.adapters.duckdb.__version__ import version as plugin_version
 from dbt.version import __version__
 
 random_logs_sql = """
@@ -138,7 +139,7 @@ def test_motherduck_user_agent(dbt_profile_target, mock_plugins, mock_creds):
             kwargs = {
                 'read_only': False,
                 'config': {
-                    'custom_user_agent': f'dbt/{__version__} downstream-dep',
+                    'custom_user_agent': f'dbt/{__version__} dbt-duckdb/{plugin_version} downstream-dep',
                     'motherduck_token': 'quack'
                 }
             }

tests/unit/test_duckdb_adapter.py

@@ -45,6 +45,7 @@ def adapter(self):
 
     @mock.patch("dbt.adapters.duckdb.environments.duckdb")
     def test_acquire_connection(self, connector):
+        connector.__version__ = "0.1.0"  # dummy placeholder for semver checks
         DuckDBConnectionManager.close_all_connections()
         connection = self.adapter.acquire_connection("dummy")