diff --git a/.github/scripts/duckdb-ref-update.sh b/.github/scripts/duckdb-ref-update.sh new file mode 100755 index 0000000..35bfcfe --- /dev/null +++ b/.github/scripts/duckdb-ref-update.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fetch_latest_ref() { + local REPO=$1 + local REF_TYPE=$2 + + if [ "$REF_TYPE" = "release" ]; then + local TAG=$(gh release -R "$REPO" list --json tagName,isLatest --jq '.[] | select(.isLatest).tagName') + local LATEST_REF=$(gh api repos/"$REPO"/git/ref/tags/"$TAG" --jq '.object.sha') + else + local LATEST_REF=$(gh api repos/"$REPO"/git/refs/heads/main --jq '.object.sha') + fi + + echo "$LATEST_REF" +} + +update_makefile() { + local REF_VAR_NAME="$1" + local LATEST_REF="$2" + + if grep -q "^.*${REF_VAR_NAME} *= *${LATEST_REF}" Makefile; then + return + fi + + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "s|^\\(.*${REF_VAR_NAME} *\\)=.*|\\1=${LATEST_REF}|" Makefile + else + sed -i "s|^\\(.*${REF_VAR_NAME} *\\)=.*|\\1=${LATEST_REF}|" Makefile + fi + + echo "\\n-Updated ${REF_VAR_NAME} to ${LATEST_REF}" +} diff --git a/.github/workflows/duckdb-ref-update.yml b/.github/workflows/duckdb-ref-update.yml new file mode 100644 index 0000000..0fd550e --- /dev/null +++ b/.github/workflows/duckdb-ref-update.yml @@ -0,0 +1,55 @@ +name: duckdb-ref-update + +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * 6' # Every Saturday at 4 AM US time (America/New_York) + +concurrency: + group: "${{ github.workflow }}-${{ github.run_started_at }}" + cancel-in-progress: true + +jobs: + update-duckdb-refs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Fetch and Update All Refs + id: fetch-update-refs + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + source ./.github/scripts/duckdb-ref-update.sh + COMMIT_DETAILS="" + + repos='[ + {"repository": "duckdb/duckdb", "ref_var": "DUCKDB_REF", "ref_type": "release"}, + {"repository": "duckdb/duckdb-aws", "ref_var": "DUCKDB_AWS_REF", "ref_type": "tip"}, + {"repository": "duckdb/duckdb-httpfs", "ref_var": "DUCKDB_HTTPFS_REF", "ref_type": "tip"}, + {"repository": "substrait-io/duckdb-substrait-extension", "ref_var": "DUCKDB_SUBSTRAIT_REF", "ref_type": "tip"}, + {"repository": "duckdb/duckdb-iceberg", "ref_var": "DUCKDB_ICEBERG_REF", "ref_type": "tip"} + ]' + + echo "$repos" | jq -c '.[]' | while read -r repo; do + REPO=$(echo "$repo" | jq -r '.repository') + REF_VAR=$(echo "$repo" | jq -r '.ref_var') + REF_TYPE=$(echo "$repo" | jq -r '.ref_type') + + echo "Processing $REPO with $REF_VAR and $REF_TYPE" + LATEST_REF=$(fetch_latest_ref "$REPO" "$REF_TYPE") + echo "Latest ref: $LATEST_REF" + UPDATE_OUTPUT=$(update_makefile "$REF_VAR" "$LATEST_REF") + if [[ -n "$UPDATE_OUTPUT" ]]; then + COMMIT_DETAILS+=$'\n'"$UPDATE_OUTPUT" + fi + echo "Commit details: $COMMIT_DETAILS" + done + + git diff -- + echo "COMMIT_DETAILS=$COMMIT_DETAILS" >> "$GITHUB_ENV" + - name: Commit and Push Updates + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: | + Update DuckDB references:${{ env.COMMIT_DETAILS }} diff --git a/Makefile b/Makefile index f26fcc1..b88c8aa 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,11 @@ DUCKDB_REPO=https://github.com/duckdb/duckdb.git DUCKDB_REF=5f5512b827df6397afd31daedb4bbdee76520019 +export DUCKDB_AWS_REF=b3050f35c6e99fa35465230493eeab14a78a0409 +export DUCKDB_HTTPFS_REF=85ac4667bcb0d868199e156f8dd918b0278db7b9 +export DUCKDB_SUBSTRAIT_REF=a1b341cc3df16c55535c12acce375040ffe50347 +export DUCKDB_ICEBERG_REF=3060b30309d82f1059c928de7280286fcf800545 + CFLAGS = -O3 CXXFLAGS = -O3 CC = "" @@ -51,7 +56,7 @@ duckdb: rm -rf duckdb git clone --depth 1 $(DUCKDB_REPO) duckdb cd duckdb && git fetch --depth 1 origin $(DUCKDB_REF) && git checkout $(DUCKDB_REF) - cp extension_config_local.cmake duckdb/extension/extension_config.cmake + envsubst < extension_config_local.cmake > duckdb/extension/extension_config.cmake .PHONY: deps.header deps.header: duckdb diff --git a/extension_config_local.cmake b/extension_config_local.cmake index 1af7607..9078ccc 100644 --- a/extension_config_local.cmake +++ b/extension_config_local.cmake @@ -8,20 +8,20 @@ duckdb_extension_load(parquet) duckdb_extension_load(icu) duckdb_extension_load(aws GIT_URL https://github.com/duckdb/duckdb-aws - GIT_TAG b3050f35c6e99fa35465230493eeab14a78a0409 + GIT_TAG ${DUCKDB_AWS_REF} ) duckdb_extension_load(httpfs GIT_URL https://github.com/duckdb/duckdb-httpfs - GIT_TAG 85ac4667bcb0d868199e156f8dd918b0278db7b9 + GIT_TAG ${DUCKDB_HTTPFS_REF} INCLUDE_DIR extension/httpfs/include ) duckdb_extension_load(tpch) duckdb_extension_load(tpcds) duckdb_extension_load(substrait GIT_URL https://github.com/substrait-io/duckdb-substrait-extension - GIT_TAG a1b341cc3df16c55535c12acce375040ffe50347 + GIT_TAG ${DUCKDB_SUBSTRAIT_REF} ) duckdb_extension_load(iceberg GIT_URL https://github.com/duckdb/duckdb-iceberg - GIT_TAG 3060b30309d82f1059c928de7280286fcf800545 + GIT_TAG ${DUCKDB_ICEBERG_REF} )