From 00f9893cba5063886a3676acfbf8ccbc2262fd54 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 22 Jul 2024 18:09:02 -0400 Subject: [PATCH 01/10] Update integration workflow --- .github/scripts/update_dev_packages.sh | 17 ++++++++++ .github/workflows/integration.yml | 44 ++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100755 .github/scripts/update_dev_packages.sh diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh new file mode 100755 index 000000000..fb26c39b5 --- /dev/null +++ b/.github/scripts/update_dev_packages.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e +set -e + + +adapters_git_branch=$1 +core_git_branch=$2 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git.*#egg=dbt-adapters|dbt-adapters.git@${adapters_git_branch}#egg=dbt-adapters|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$adapters_req_sed_pattern" $target_req_file +else + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$adapters_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 03c1e06ed..a4138b50d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,8 @@ name: Adapter Integration Tests +run-name: ${{ inputs.name }} by @${{ github.actor }} + on: # pushes to release branches push: @@ -34,10 +36,39 @@ on: # manual trigger workflow_dispatch: inputs: - dbt-core-branch: + dbt_core_branch: description: "branch of dbt-core to use in dev-requirements.txt" required: false type: string + default: main + name: + description: "name to associate with run" + required: false + type: string + default: "Adapter Integration Tests" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: main + workflow_call: + inputs: + dbt_core_branch: + description: "branch of dbt-core to use in dev-requirements.txt" + required: false + type: string + default: main + name: + description: "name to associate with run" + required: false + type: string + default: "Adapter Integration Tests" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: main + # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -148,7 +179,7 @@ jobs: with: persist-credentials: false - # explicity checkout the branch for the PR, + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' @@ -169,11 +200,12 @@ jobs: python -m pip --version tox --version - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }} run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + ./.github/scripts/update_dev_packages.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} - name: Run tox (bigquery) if: matrix.adapter == 'bigquery' From 4d7e2f445edd0b6f828a4c3ea51bba1ea9cb7e6a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 16:52:07 -0400 Subject: [PATCH 02/10] Add "adapter_branch" --- .github/workflows/integration.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a4138b50d..b4eddb82a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ name: Adapter Integration Tests -run-name: ${{ inputs.name }} by @${{ github.actor }} +run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ contains(github.event_name, 'workflow_') && inputs.adapter_branch || github.ref_name }} by @${{ github.actor }}" on: # pushes to release branches @@ -51,6 +51,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main workflow_call: inputs: dbt_core_branch: @@ -68,6 +73,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main # explicitly turn off permissions for `GITHUB_TOKEN` @@ -98,11 +108,20 @@ jobs: run-python-tests: ${{ steps.filter.outputs.bigquery-python }} steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' + - name: Check out the repository + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' uses: actions/checkout@v4 with: persist-credentials: false + # We can't actually change branch until this is in main, because it + # won't find the update_dev_packages.sh script. + # ref: ${{ inputs.adapter_branch }} - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' From 3575be366ae354d5074c5aa1b8fcc5df37e34973 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 10:51:55 -0400 Subject: [PATCH 03/10] Update update_dev_packages.sh --- .github/scripts/update_dev_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh index fb26c39b5..7fdd47c35 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_packages.sh @@ -6,7 +6,7 @@ adapters_git_branch=$1 core_git_branch=$2 target_req_file="dev-requirements.txt" core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" -adapters_req_sed_pattern="s|dbt-adapters.git.*#egg=dbt-adapters|dbt-adapters.git@${adapters_git_branch}#egg=dbt-adapters|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file From 578c47825d5043ea2bc9658c853912563aeab307 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 16:55:33 -0400 Subject: [PATCH 04/10] cat dev-requirements.txt --- .github/workflows/integration.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c65dc7a84..618ed689a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -193,11 +193,20 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' uses: actions/checkout@v4 with: persist-credentials: false + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + uses: actions/checkout@v4 + with: + persist-credentials: false + # We can't actually change branch until this is in main, because it + # won't find the update_dev_packages.sh script. + # ref: ${{ inputs.adapter_branch }} + # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event - name: Check out the repository (PR) @@ -225,6 +234,7 @@ jobs: ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} + cat dev-requirements.txt - name: Run tox (bigquery) if: matrix.adapter == 'bigquery' From 8d80ba4aa7942dba9b2e1c5d9f5bf2fa08842a75 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 17:45:06 -0400 Subject: [PATCH 05/10] Remove workflow_call --- .github/workflows/integration.yml | 33 +++++-------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 618ed689a..7413b7917 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -56,29 +56,6 @@ on: type: string required: false default: main - workflow_call: - inputs: - dbt_core_branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - default: main - name: - description: "name to associate with run" - required: false - type: string - default: "Adapter Integration Tests" - dbt_adapters_branch: - description: "The branch of dbt-adapters to use" - type: string - required: false - default: main - adapter_branch: - description: "The branch of this adapter repository to use" - type: string - required: false - default: main - # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -109,13 +86,13 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false @@ -193,13 +170,13 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false @@ -229,7 +206,7 @@ jobs: tox --version - name: Update Adapters and Core branches (update dev_requirements.txt) - if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }} + if: github.event_name == 'workflow_dispatch' run: | ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ From 7b28bce502fe1b9ba037a4e5fd51aaf763b5a678 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 25 Jul 2024 17:00:28 -0400 Subject: [PATCH 06/10] Remove update_dbt_core_branch.sh --- .github/scripts/update_dbt_core_branch.sh | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100755 .github/scripts/update_dbt_core_branch.sh diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major From 41cfdd39cd11e9e55619a44decb34a2d9a01e8a2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 25 Jul 2024 17:12:45 -0400 Subject: [PATCH 07/10] Use adapter_branch --- .github/workflows/integration.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7413b7917..33f3c948c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -96,9 +96,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - # We can't actually change branch until this is in main, because it - # won't find the update_dev_packages.sh script. - # ref: ${{ inputs.adapter_branch }} + ref: ${{ inputs.adapter_branch }} - name: Check out the repository (PR) if: github.event_name == 'pull_request_target' @@ -180,9 +178,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - # We can't actually change branch until this is in main, because it - # won't find the update_dev_packages.sh script. - # ref: ${{ inputs.adapter_branch }} + ref: ${{ inputs.adapter_branch }} # explicitly checkout the branch for the PR, # this is necessary for the `pull_request_target` event From b17f24a3990793259de598d76a70c4e2e3d0c5ff Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 11:08:43 -0400 Subject: [PATCH 08/10] Change script name, add dbt_common --- ...s.sh => update_dev_dependency_branches.sh} | 16 ++++++---- .github/workflows/integration.yml | 32 +++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) rename .github/scripts/{update_dev_packages.sh => update_dev_dependency_branches.sh} (62%) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_dependency_branches.sh similarity index 62% rename from .github/scripts/update_dev_packages.sh rename to .github/scripts/update_dev_dependency_branches.sh index 7fdd47c35..022df6a8a 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -2,16 +2,20 @@ set -e -adapters_git_branch=$1 -core_git_branch=$2 +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" -adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file else - sed -i "$core_req_sed_pattern" $target_req_file sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 33f3c948c..d3980a080 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ name: Adapter Integration Tests -run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ contains(github.event_name, 'workflow_') && inputs.adapter_branch || github.ref_name }} by @${{ github.actor }}" +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" on: # pushes to release branches @@ -36,26 +36,31 @@ on: # manual trigger workflow_dispatch: inputs: - dbt_core_branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - default: main name: - description: "name to associate with run" + description: "Name to associate with run (example: 'dbt-adapters-242')" required: false type: string default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main dbt_adapters_branch: description: "The branch of dbt-adapters to use" type: string required: false - default: main - adapter_branch: - description: "The branch of this adapter repository to use" + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" type: string required: false - default: main + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -204,9 +209,10 @@ jobs: - name: Update Adapters and Core branches (update dev_requirements.txt) if: github.event_name == 'workflow_dispatch' run: | - ./.github/scripts/update_dev_packages.sh \ + ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ - ${{ inputs.dbt_core_branch }} + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} cat dev-requirements.txt - name: Run tox (bigquery) From c233b45996d0926cea388532803b6f4ae457c428 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 13:51:06 -0400 Subject: [PATCH 09/10] Consistent quotes for defaults --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d3980a080..afeb214a3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -45,7 +45,7 @@ on: description: "The branch of this adapter repository to use" type: string required: false - default: main + default: "main" dbt_adapters_branch: description: "The branch of dbt-adapters to use" type: string From 18d258e19d683182d9b6ed65d5fa5366639b64dd Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 15:09:42 -0400 Subject: [PATCH 10/10] Tweak if for repository checkout --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index afeb214a3..ae75a4f2a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -90,8 +90,8 @@ jobs: run-python-tests: ${{ steps.filter.outputs.bigquery-python }} steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' + - name: Check out the repository (push) + if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false