add workflow for updating duckdb refs

.github/scripts/duckdb-ref-update.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#set -euo pipefail
+
+fetch_latest_ref() {
+  # Fetch the latest reference SHA based on REF_TYPE (release or tip)
+  local REPO=$1
+  local REF_TYPE=$2
+
+  if [ "$REF_TYPE" = "release" ]; then
+    local TAG=$(gh release -R "$REPO" list --json tagName,isLatest --jq '.[] | select(.isLatest).tagName')
+    local LATEST_REF=$(gh api repos/"$REPO"/git/ref/tags/"$TAG" --jq '.object.sha')
+  else
+    local LATEST_REF=$(gh api repos/"$REPO"/git/refs/heads/main --jq '.object.sha')
+  fi
+
+  echo "$LATEST_REF"
+}
+
+update_makefile() {
+  # Update the Makefile with the new reference for the given variable
+  local REF_VAR_NAME="$1"
+  local LATEST_REF="$2"
+
+  # Replace the entire line correctly, preserving the original prefix
+  if [[ "$(uname)" == "Darwin" ]]; then
+    sed -i '' "s|^\\(.*${REF_VAR_NAME} *\\)=.*|\\1= ${LATEST_REF}|" Makefile
+  else
+    sed -i "s|^\\(.*${REF_VAR_NAME} *\\)=.*|\\1= ${LATEST_REF}|" Makefile
+  fi
+
+  return 0
+}

.github/workflows/duckdb-ref-update.yml

@@ -0,0 +1,54 @@
+name: duckdb-ref-update
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  schedule:
+    - cron: '0 4 * * 6'  # Every Saturday at 4 AM US time (America/New_York)
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.run_started_at }}"
+  cancel-in-progress: true
+
+jobs:
+  update-duckdb-refs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fetch and Update All Refs
+        id: fetch-update-refs
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          source ./.github/scripts/duckdb-ref-update.sh
+          COMMIT_DETAILS=""
+
+          repos='[
+            {"repository": "duckdb/duckdb", "env_var": "DUCKDB_REF", "ref_type": "release"},
+            {"repository": "duckdb/duckdb-aws", "env_var": "DUCKDB_AWS_REF", "ref_type": "tip"},
+            {"repository": "duckdb/duckdb-httpfs", "env_var": "DUCKDB_HTTPFS_REF", "ref_type": "tip"},
+            {"repository": "substrait-io/duckdb-substrait-extension", "env_var": "DUCKDB_SUBSTRAIT_REF", "ref_type": "tip"},
+            {"repository": "duckdb/duckdb-iceberg", "env_var": "DUCKDB_ICEBERG_REF", "ref_type": "tip"}
+          ]'
+
+          echo "$repos" | jq -c '.[]' | while read -r repo; do
+            REPO=$(echo "$repo" | jq -r '.repository')
+            ENV_VAR=$(echo "$repo" | jq -r '.env_var')
+            REF_TYPE=$(echo "$repo" | jq -r '.ref_type')
+
+            LATEST_REF=$(fetch_latest_ref "$REPO" "$REF_TYPE")
+            if update_makefile "$ENV_VAR" "$LATEST_REF"; then
+              COMMIT_DETAILS+="\\n- Updated $REPO to $LATEST_REF"
+            fi
+          done
+
+          echo "COMMIT_DETAILS=$COMMIT_DETAILS" >> "$GITHUB_ENV"
+
+      - name: Commit and Push Updates
+        if: env.COMMIT_DETAILS != ''
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: |
+            Update DuckDB references:${{ env.COMMIT_DETAILS }}

Makefile

@@ -1,5 +1,10 @@
 DUCKDB_REPO=https://github.com/duckdb/duckdb.git
-DUCKDB_REF=5f5512b827df6397afd31daedb4bbdee76520019
+DUCKDB_REF = kittu
+
+export DUCKDB_AWS_REF = kittu
+export DUCKDB_HTTPFS_REF = kittu
+export DUCKDB_SUBSTRAIT_REF = a1b341cc3df16c55535c12acce375040ffe50347
+export DUCKDB_ICEBERG_REF = 3060b30309d82f1059c928de7280286fcf800545
 
 CFLAGS   = -O3
 CXXFLAGS = -O3
@@ -51,7 +56,7 @@ duckdb:
 	rm -rf duckdb
 	git clone --depth 1 $(DUCKDB_REPO) duckdb
 	cd duckdb && git fetch --depth 1 origin $(DUCKDB_REF) && git checkout $(DUCKDB_REF)
-	cp extension_config_local.cmake duckdb/extension/extension_config.cmake
+	envsubst < extension_config_local.cmake > duckdb/extension/extension_config.cmake
 
 .PHONY: deps.header
 deps.header: duckdb

extension_config_local.cmake

@@ -8,20 +8,20 @@ duckdb_extension_load(parquet)
 duckdb_extension_load(icu)
 duckdb_extension_load(aws
     GIT_URL https://github.com/duckdb/duckdb-aws
-    GIT_TAG b3050f35c6e99fa35465230493eeab14a78a0409
+    GIT_TAG ${DUCKDB_AWS_REF}
 )
 duckdb_extension_load(httpfs
     GIT_URL https://github.com/duckdb/duckdb-httpfs
-    GIT_TAG 85ac4667bcb0d868199e156f8dd918b0278db7b9
+    GIT_TAG ${DUCKDB_HTTPFS_REF}
     INCLUDE_DIR extension/httpfs/include
 )
 duckdb_extension_load(tpch)
 duckdb_extension_load(tpcds)
 duckdb_extension_load(substrait
     GIT_URL https://github.com/substrait-io/duckdb-substrait-extension
-    GIT_TAG a1b341cc3df16c55535c12acce375040ffe50347
+    GIT_TAG ${DUCKDB_SUBSTRAIT_REF}
 )
 duckdb_extension_load(iceberg
     GIT_URL https://github.com/duckdb/duckdb-iceberg
-    GIT_TAG 3060b30309d82f1059c928de7280286fcf800545
+    GIT_TAG ${DUCKDB_ICEBERG_REF}
 )