diff --git a/.github/workflows/assign_milestone.yml b/.github/workflows/assign_milestone.yml index 13c66e828df..c907167eb3e 100644 --- a/.github/workflows/assign_milestone.yml +++ b/.github/workflows/assign_milestone.yml @@ -4,7 +4,9 @@ on: pull_request_target: types: [opened] -permissions: read-all +permissions: + pull-requests: write + contents: read env: GH_TOKEN: ${{ github.token }} @@ -13,10 +15,9 @@ jobs: build: name: Assign Milestone runs-on: ubuntu-24.04 - permissions: - pull-requests: write steps: +<<<<<<< HEAD - name: Set up Go uses: actions/setup-go@v5 with: @@ -24,7 +25,24 @@ jobs: - name: Checkout code uses: actions/checkout@v4 +======= + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + # We must explicitly checkout the base's SHA to avoid executing any code coming from + # the PR's SHA - Which would be executed in the base branch's context. + # This is really important to limit any sort of pwn requests. + ref: ${{ github.base_ref }} + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Assign Milestone run: | - gh pr edit ${{ github.event.number }} --milestone "v$(sed -n 's/.*versionName.*\"\([[:digit:]\.]*\).*\"/\1/p' ./go/vt/servenv/version.go)" + # Ensure the content we sed from version.go is sanitized and match the correct format + VERSION=$(sed -n 's/.*versionName.*\"\([[:digit:]\.]*\).*\"/\1/p' ./go/vt/servenv/version.go) + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid version format: $VERSION" + exit 1 + fi + + gh pr edit ${{ github.event.number }} --milestone "v$VERSION" diff --git a/.github/workflows/auto_approve_pr.yml b/.github/workflows/auto_approve_pr.yml index c6a07ad9965..7c1578ad1d1 100644 --- a/.github/workflows/auto_approve_pr.yml +++ b/.github/workflows/auto_approve_pr.yml @@ -16,7 +16,14 @@ jobs: steps: - name: Checkout code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Auto Approve Pull Request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check_make_vtadmin_authz_testgen.yml b/.github/workflows/check_make_vtadmin_authz_testgen.yml index 17f1f3de384..d9d01e7cb15 100644 --- a/.github/workflows/check_make_vtadmin_authz_testgen.yml +++ b/.github/workflows/check_make_vtadmin_authz_testgen.yml @@ -27,7 +27,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/check_make_vtadmin_web_proto.yml b/.github/workflows/check_make_vtadmin_web_proto.yml index f7adbfd67f5..2034ccdcff7 100644 --- a/.github/workflows/check_make_vtadmin_web_proto.yml +++ b/.github/workflows/check_make_vtadmin_web_proto.yml @@ -27,7 +27,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index db6af355d59..711b85e89fd 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (12) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index 55ad0e07f6f..b79dcb5a711 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (13) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index b3068fb664b..b0d4211ec83 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (15) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index 3017018ed11..e1b62865999 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (18) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index 375b31734bf..eb0435b4770 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (21) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_22.yml b/.github/workflows/cluster_endtoend_22.yml index 87915fe4754..45f9d7e08aa 100644 --- a/.github/workflows/cluster_endtoend_22.yml +++ b/.github/workflows/cluster_endtoend_22.yml @@ -15,7 +15,12 @@ env: jobs: build: +<<<<<<< HEAD:.github/workflows/cluster_endtoend_22.yml name: Run endtoend tests on Cluster (22) +======= + timeout-minutes: 60 + name: Run endtoend tests on Cluster (vtgate_plantests) +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/cluster_endtoend_vtgate_plantests.yml runs-on: ubuntu-24.04 steps: @@ -45,7 +50,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD:.github/workflows/cluster_endtoend_22.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/cluster_endtoend_vtgate_plantests.yml - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +102,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_backup_pitr.yml b/.github/workflows/cluster_endtoend_backup_pitr.yml index fdd0094c479..7f70b81a722 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (backup_pitr) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml index b21da02517f..e7e4c4d2e28 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml @@ -45,7 +45,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD:.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/unit_test_mysql84.yml - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml index 4a8c6e6eded..60144842078 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (backup_pitr_xtrabackup) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Setup Percona Server for MySQL 8.0 diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml index c4de57a9d69..1e3fc66151a 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml @@ -49,7 +49,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD:.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/unit_test_evalengine_mysql84.yml - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index 5e86c223ca0..f1d4bde3088 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (ers_prs_newfeatures_heavy) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 92d333a0f4f..d29a6ad69a9 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (mysql80) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_mysql_server_vault.yml b/.github/workflows/cluster_endtoend_mysql_server_vault.yml index 9c644c1a2cf..0a009b9d156 100644 --- a/.github/workflows/cluster_endtoend_mysql_server_vault.yml +++ b/.github/workflows/cluster_endtoend_mysql_server_vault.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (mysql_server_vault) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml index 72c05e03984..fd9a6927c54 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml @@ -15,7 +15,12 @@ env: jobs: build: +<<<<<<< HEAD:.github/workflows/cluster_endtoend_onlineddl_ghost.yml name: Run endtoend tests on Cluster (onlineddl_ghost) +======= + timeout-minutes: 60 + name: Run endtoend tests on Cluster (backup_pitr_mysqlshell) +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml runs-on: ubuntu-24.04 steps: @@ -45,7 +50,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD:.github/workflows/cluster_endtoend_onlineddl_ghost.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +103,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml index 5bf4a169567..c5453476d01 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml @@ -45,7 +45,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD:.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/unit_test_evalengine_mysql80.yml - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 49f55504efd..220ee42a8a2 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (onlineddl_revert) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index e2957b2d4f6..37fc8785b1b 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (onlineddl_scheduler) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index f86ac2dac87..7953408e8a7 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (onlineddl_vrepl) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index cc7326692ea..504e3c9329f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index 47cfe54c5a9..e4f4b18da6d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 1ad28d3a7d7..668a0bb6408 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index e9a25b426e2..003cc8139a3 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (schemadiff_vrepl) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index c20feaae478..ab362944391 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (tabletmanager_consul) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 540e4b7477e..f988ba51569 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (tabletmanager_tablegc) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml index 8dd8ee094b2..64649683609 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (tabletmanager_throttler_topo) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index 227b851460f..ac65997f065 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (topo_connection_cache) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index 00e54a2e25b..de4749f09e9 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vreplication_across_db_versions) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index af20906c462..93893a0f9bd 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vreplication_basic) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index 4bf47afa89f..36a3a8f6585 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vreplication_cellalias) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml new file mode 100644 index 00000000000..b0cac4d620e --- /dev/null +++ b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml @@ -0,0 +1,196 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vreplication_copy_parallel) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_copy_parallel)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + timeout-minutes: 60 + name: Run endtoend tests on Cluster (vreplication_copy_parallel) + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql8026.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-transaction-compression=ON + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml index 32b1d27bbca..c15877dba5b 100644 --- a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml +++ b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vreplication_foreign_key_stress) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml new file mode 100644 index 00000000000..96746b5b383 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml @@ -0,0 +1,196 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vreplication_mariadb_to_mysql) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_mariadb_to_mysql)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + timeout-minutes: 60 + name: Run endtoend tests on Cluster (vreplication_mariadb_to_mysql) + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql8026.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-transaction-compression=ON + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml index 615db1cb252..111a4c353c9 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml @@ -15,7 +15,12 @@ env: jobs: build: +<<<<<<< HEAD:.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml name: Run endtoend tests on Cluster (vreplication_migrate_vdiff2_convert_tz) +======= + timeout-minutes: 60 + name: Run endtoend tests on Cluster (vreplication_migrate) +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/cluster_endtoend_vreplication_migrate.yml runs-on: gh-hosted-runners-16cores-1-24.04 steps: @@ -45,7 +50,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD:.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/cluster_endtoend_vreplication_migrate.yml - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +102,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml new file mode 100644 index 00000000000..f97c5b48ed9 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml @@ -0,0 +1,196 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vreplication_multi_tenant) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_multi_tenant)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + timeout-minutes: 60 + name: Run endtoend tests on Cluster (vreplication_multi_tenant) + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql8026.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-transaction-compression=ON + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml index b9e266f222b..033cc650a10 100644 --- a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml +++ b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vreplication_partial_movetables_and_materialize) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index ec7a944fdf3..9c211ba7672 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vreplication_v2) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml new file mode 100644 index 00000000000..4b066740b5e --- /dev/null +++ b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml @@ -0,0 +1,196 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vreplication_vtctldclient_vdiff2_movetables_tz) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_vtctldclient_vdiff2_movetables_tz)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + timeout-minutes: 60 + name: Run endtoend tests on Cluster (vreplication_vtctldclient_vdiff2_movetables_tz) + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql8026.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-transaction-compression=ON + EOF + + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_vdiff2_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/cluster_endtoend_vstream.yml b/.github/workflows/cluster_endtoend_vstream.yml index dd3691c24d1..95597ef99a8 100644 --- a/.github/workflows/cluster_endtoend_vstream.yml +++ b/.github/workflows/cluster_endtoend_vstream.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vstream) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtbackup.yml b/.github/workflows/cluster_endtoend_vtbackup.yml index 04681a5f5e3..9eead9536f7 100644 --- a/.github/workflows/cluster_endtoend_vtbackup.yml +++ b/.github/workflows/cluster_endtoend_vtbackup.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtbackup) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index 69e070413f4..2dde020510b 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtctlbackup_sharded_clustertest_heavy) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index 283d143f46a..fdb6104776a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_concurrentdml) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml index e0f4eacc538..1aca185e27b 100644 --- a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml +++ b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_foreignkey_stress) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index 1587ca1ef15..2bf34648910 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_gen4) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index 78e83987a3c..95abf1c1593 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_general_heavy) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index 41aa8dfe536..5e95f4d0218 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_godriver) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml index 8a91c0ed580..738c16f347a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml +++ b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_partial_keyspace) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index 1385a3cc68c..e83bc7c3d01 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_queries) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index b83e133052f..44e0a10a986 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_readafterwrite) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index d89da6ecf8d..b4868693a2a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_reservedconn) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 9c5d059067b..d6eb66d1634 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_schema) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index 8a18b0aad2b..99658ed21e7 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_schema_tracker) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index ab084b94da9..c31d08a6deb 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_tablet_healthcheck_cache) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index da0d9ac3d17..987ab0bf762 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_topo) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index 0581024c649..91b2474046f 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_topo_consul) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index ff10515a841..81f58b2ae02 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_topo_etcd) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index ea6857aa831..02061ef4790 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_transaction) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index 349051b76e1..99ce3196776 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_unsharded) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index 496a732a68f..0a27d5289b4 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_vindex_heavy) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index 07324fe0873..a8a611790b5 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtgate_vschema) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 40da4a208e3..1cee46f6a69 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vtorc) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml index 293600e79f8..257938c7c40 100644 --- a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml +++ b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (vttablet_prscomplex) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Get key to latest MySQL repo diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index faccfd17507..7a0ce45bcbf 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (xb_backup) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Setup Percona Server for MySQL 8.0 diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index 79c156f1c5f..e2eed8650c7 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -15,6 +15,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on Cluster (xb_recovery) runs-on: ubuntu-24.04 @@ -45,7 +46,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -91,6 +98,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | # Setup Percona Server for MySQL 8.0 diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 956b908d6ba..6cbaef14fff 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -13,7 +13,13 @@ jobs: steps: - name: Check out code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in files relevant to code coverage uses: dorny/paths-filter@v3.0.1 diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index 93530236a10..e8cc9b1ad3b 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -27,7 +27,13 @@ jobs: steps: - name: Checkout repository +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Set up Go uses: actions/setup-go@v5 @@ -47,6 +53,7 @@ jobs: # queries: security-extended,security-and-quality - name: Get base dependencies + timeout-minutes: 10 run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Uninstall any previously installed MySQL first diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 20da0a451d9..aee668cc3eb 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -7,25 +7,37 @@ on: release: types: [created] -permissions: read-all +permissions: + contents: write + actions: read jobs: build: name: Create Release runs-on: ubuntu-24.04 - permissions: - contents: write steps: +<<<<<<< HEAD +======= + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Set up Go uses: actions/setup-go@v5 with: +<<<<<<< HEAD go-version: 1.22.10 - name: Setup node uses: actions/setup-node@v4 with: node-version: '18.16.0' +======= + go-version-file: go.mod + cache: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Tune the OS run: | @@ -39,14 +51,36 @@ jobs: sudo apt-get update sudo apt-get install -y make ruby ruby-dev go mod download - sudo gem install fpm + + # We use fpm to package our artifacts, we want to pin the version we use and + # ensure the checksum of the package matches the one published on the package's webpage. + # https://rubygems.org/gems/fpm/versions + - name: Get fpm + run: | + FPM_VERSION=1.16.0 + gem fetch fpm -v $FPM_VERSION + + # Reviewers: The expected checksum MUST ALWAYS match the one published on this website: + # https://rubygems.org/gems/fpm/versions + EXPECTED_CHECKSUM="d9eafe613cfbdf9d3b8ef2e321e194cd0a2d300ce37f716c0be1b3a42b7db5df" + + GOT_CHECKSUM=$(sha256sum fpm-$FPM_VERSION.gem | awk '{print $1}') + + if [[ "$GOT_CHECKSUM" != "$EXPECTED_CHECKSUM" ]]; then + echo "Checksum validation failed" + echo "Expected: $EXPECTED_CHECKSUM" + echo "Got: $GOT_CHECKSUM" + exit 1 + fi + + sudo gem install fpm-$FPM_VERSION.gem - name: Make Packages run: | ./tools/make-release-packages.sh - name: Upload Files - uses: csexton/release-asset-action@master + uses: csexton/release-asset-action@3567794e918fa3068116688122a76cdeb57b5f09 # v3.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} pattern: "releases/*.{tar.gz,rpm,deb}" diff --git a/.github/workflows/docker_build_base.yml b/.github/workflows/docker_build_base.yml index de81622749d..9a2145d7e4c 100644 --- a/.github/workflows/docker_build_base.yml +++ b/.github/workflows/docker_build_base.yml @@ -21,11 +21,76 @@ jobs: strategy: fail-fast: true matrix: +<<<<<<< HEAD:.github/workflows/docker_build_base.yml +======= + branch: [ mysql80 ] + + steps: + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on main + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/vttestserver:${{ matrix.branch }} + + ###### + # All code below only applies to new tags + ###### + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} + + build_and_push_lite: + name: Build and push lite + runs-on: ubuntu-24.04 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml branch: [ latest, percona80 ] steps: - name: Check out code +<<<<<<< HEAD:.github/workflows/docker_build_base.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml - name: Login to Docker Hub uses: docker/login-action@v3 @@ -42,8 +107,13 @@ jobs: fi - name: Build and push on main +<<<<<<< HEAD:.github/workflows/docker_build_base.yml if: github.ref == 'refs/heads/main' uses: docker/build-push-action@v5 +======= + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml with: context: . file: ${{ env.DOCKERFILE }} @@ -66,8 +136,13 @@ jobs: fi - name: Build and push on tags +<<<<<<< HEAD:.github/workflows/docker_build_base.yml if: startsWith(github.ref, 'refs/tags/') && matrix.branch == 'latest' uses: docker/build-push-action@v5 +======= + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml with: context: . file: ${{ env.DOCKERFILE }} @@ -87,7 +162,13 @@ jobs: steps: - name: Check out code +<<<<<<< HEAD:.github/workflows/docker_build_base.yml uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml - name: Login to Docker Hub uses: docker/login-action@v3 @@ -100,6 +181,7 @@ jobs: echo "DOCKER_CTX=./docker/k8s" >> $GITHUB_ENV - name: Build and push on main latest tag +<<<<<<< HEAD:.github/workflows/docker_build_base.yml if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' uses: docker/build-push-action@v5 with: @@ -196,6 +278,10 @@ jobs: - name: Build and push on main latest tag if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' uses: docker/build-push-action@v5 +======= + if: startsWith(github.ref, 'refs/tags/') == false && matrix.debian == 'bookworm' + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml with: context: ${{ env.DOCKER_CTX }} push: true @@ -205,8 +291,13 @@ jobs: DEBIAN_VER=${{ matrix.debian }}-slim - name: Build and push on main debian specific tag +<<<<<<< HEAD:.github/workflows/docker_build_base.yml if: github.ref == 'refs/heads/main' uses: docker/build-push-action@v5 +======= + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml with: context: ${{ env.DOCKER_CTX }} push: true @@ -238,7 +329,11 @@ jobs: # Build and Push component image to DOCKER_TAG, applies to both debian version - name: Build and push on tags using Debian extension if: startsWith(github.ref, 'refs/tags/') +<<<<<<< HEAD:.github/workflows/docker_build_base.yml uses: docker/build-push-action@v5 +======= + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml with: context: ${{ env.DOCKER_CTX }} push: true @@ -251,7 +346,11 @@ jobs: # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already - name: Build and push on tags without Debian extension if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm' +<<<<<<< HEAD:.github/workflows/docker_build_base.yml uses: docker/build-push-action@v5 +======= + uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)):.github/workflows/docker_build_images.yml with: context: ${{ env.DOCKER_CTX }} push: true diff --git a/.github/workflows/docker_test_cluster_10.yml b/.github/workflows/docker_test_cluster_10.yml index 34ad64bfdd0..0bb1ab97798 100644 --- a/.github/workflows/docker_test_cluster_10.yml +++ b/.github/workflows/docker_test_cluster_10.yml @@ -27,7 +27,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/docker_test_cluster_25.yml b/.github/workflows/docker_test_cluster_25.yml index 00708e113da..105fbe95683 100644 --- a/.github/workflows/docker_test_cluster_25.yml +++ b/.github/workflows/docker_test_cluster_25.yml @@ -27,7 +27,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 632b1601d3b..5310237f6f6 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -26,7 +26,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml index 0136d81ab04..216db0e1f1f 100644 --- a/.github/workflows/endtoend.yml +++ b/.github/workflows/endtoend.yml @@ -26,7 +26,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index 8f8df787760..fe11f9517eb 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -30,7 +30,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index af80a712014..5f6794293e0 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -30,7 +30,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/static_checks_etc.yml b/.github/workflows/static_checks_etc.yml index 916c20d6ec2..c974eaabcb1 100644 --- a/.github/workflows/static_checks_etc.yml +++ b/.github/workflows/static_checks_etc.yml @@ -31,7 +31,13 @@ jobs: - name: Checkout code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Run FOSSA scan and upload build data # Fails on pull requests when using the API key secret. diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml index 5556221cc02..d3b6c67f4b2 100644 --- a/.github/workflows/unit_race.yml +++ b/.github/workflows/unit_race.yml @@ -31,7 +31,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/unit_race_evalengine.yml b/.github/workflows/unit_race_evalengine.yml new file mode 100644 index 00000000000..70523aa2b8a --- /dev/null +++ b/.github/workflows/unit_race_evalengine.yml @@ -0,0 +1,155 @@ +name: unit_race_evalengine +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'unit_race_evalengine') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + + build: + name: Unit Test (Evalengine_Race) + runs-on: gh-hosted-runners-16cores-1-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + unit_tests: + - 'go/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/unit_race_evalengine.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get -qq update + + # mysql80 + sudo apt-get -qq install -y mysql-server mysql-client + + sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo service mysql stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + mkdir -p dist bin + curl -s -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist + mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/ + + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Run make tools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + make tools + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: unit_race_evalengine + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + export NOVTADMINBUILD=1 + export VTEVALENGINETEST="1" + + eatmydata -- make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/unit_test_evalengine_mysql57.yml b/.github/workflows/unit_test_evalengine_mysql57.yml new file mode 100644 index 00000000000..de98a6779d2 --- /dev/null +++ b/.github/workflows/unit_test_evalengine_mysql57.yml @@ -0,0 +1,190 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Unit Test (evalengine_mysql57) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Unit Test (evalengine_mysql57)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + test: + name: Unit Test (evalengine_mysql57) + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + unit_tests: + - 'go/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/unit_test_evalengine_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + + # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 + # packages for Jammy. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + # We have to install this old version of libaio1. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 + + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo service mysql stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + mkdir -p dist bin + curl -s -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist + mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/ + + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Run make tools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + make tools + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + timeout-minutes: 30 + run: | + set -exo pipefail + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + + export NOVTADMINBUILD=1 + export VTEVALENGINETEST="1" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql57" + + eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index 347ac17ff3c..60b7228f427 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -45,7 +45,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 52602a3006b..4dd9a1a5633 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -45,7 +45,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/update_golang_dependencies.yml b/.github/workflows/update_golang_dependencies.yml index 80b1fa3c44e..a6b80f1e94e 100644 --- a/.github/workflows/update_golang_dependencies.yml +++ b/.github/workflows/update_golang_dependencies.yml @@ -22,9 +22,14 @@ jobs: go-version: 1.22.10 - name: Check out code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: main + persist-credentials: 'false' - name: Upgrade the Golang Dependencies id: detect-and-update diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index e13279a2a31..aa92cd95444 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -25,7 +25,11 @@ jobs: go-version: 1.22.10 - name: Check out code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ matrix.branch }} diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index 32427bc6020..b9978444128 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -35,9 +35,14 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -84,6 +89,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update @@ -107,9 +113,14 @@ jobs: # Checkout to the last release of Vitess - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -129,7 +140,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index c8903f57160..22bff6376a6 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -25,9 +25,14 @@ jobs: fi - name: Check out commit's code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-next-release-ref @@ -86,6 +91,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update @@ -109,9 +115,14 @@ jobs: # Checkout to the next release of Vitess - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -131,7 +142,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index 76efbdffecd..d7659e31556 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -38,9 +38,14 @@ jobs: # Checkout to this build's commit - name: Checkout to commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-previous-release-ref @@ -88,6 +93,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -129,9 +135,14 @@ jobs: # Checkout to the last release of Vitess - name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -151,7 +162,13 @@ jobs: # Checkout to this build's commit - name: Checkout to commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml index b889b490b25..2127af9ec99 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml @@ -27,9 +27,14 @@ jobs: # Checkout to this build's commit - name: Checkout to commit's code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-next-release-ref @@ -89,6 +94,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -130,9 +136,14 @@ jobs: # Checkout to the next release of Vitess - name: Checkout to the other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -152,7 +163,13 @@ jobs: # Checkout to this build's commit - name: Checkout to commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml new file mode 100644 index 00000000000..12fd8735051 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml @@ -0,0 +1,262 @@ +name: Online DDL flow - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Online DDL flow') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our Online DDL + VReplication + throttler components +# work using primary and replica vttablets built on different versions. + +jobs: + + upgrade_downgrade_test: + timeout-minutes: 60 + name: Run Upgrade Downgrade Test - Online DDL flow + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml' + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Set output with next release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.23.4 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + timeout-minutes: 10 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the last release of Vitess + - name: Check out last version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + echo "building last release: $(git rev-parse HEAD)" + source build.env + make build + mkdir -p /tmp/vitess-build-last/ + cp -R bin /tmp/vitess-build-last/ + rm -Rf bin/* + + # Checkout to the next release of Vitess + - name: Check out next version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + echo "building next release: $(git rev-parse HEAD)" + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-next/ + cp -R bin /tmp/vitess-build-next/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + echo "building this SHA: $(git rev-parse HEAD)" + source build.env + make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Copy vttablet and related binaries under new names + - name: Use current version Vtctl, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp /tmp/vitess-build-last/bin/vttablet $PWD/bin/vttablet-last + cp /tmp/vitess-build-last/bin/mysqlctl $PWD/bin/mysqlctl-last + cp /tmp/vitess-build-last/bin/mysqlctld $PWD/bin/mysqlctld-last + cp /tmp/vitess-build-next/bin/vttablet $PWD/bin/vttablet-next + cp /tmp/vitess-build-next/bin/mysqlctl $PWD/bin/mysqlctl-next + cp /tmp/vitess-build-next/bin/mysqlctld $PWD/bin/mysqlctld-next + $PWD/bin/vttablet-last --version + $PWD/bin/vttablet --version + $PWD/bin/vttablet-next --version + + # Running a test with primary tablet at version n (current SHA) and replica vttablet at version n-1 + - name: Run Online DDL tests (primary=N, replica=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="" + export REPLICA_TABLET_BINARY_SUFFIX="-last" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + + # Running a test with primary tablet at version n-1 and replica vttablet at version n (current SHA) + - name: Run Online DDL tests (primary=N-1, replica=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="-last" + export REPLICA_TABLET_BINARY_SUFFIX="" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + + # Running a test with primary tablet at version n+1 and replica vttablet at version n (current SHA) + - name: Run Online DDL tests (primary=N+1, replica=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="-next" + export REPLICA_TABLET_BINARY_SUFFIX="" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + + # Running a test with primary tablet at version n (current SHA) and replica vttablet at version n+1 + - name: Run Online DDL tests (primary=N, replica=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="" + export REPLICA_TABLET_BINARY_SUFFIX="-next" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml index e00c13b4a7f..bf3d30c7f72 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Query Serving (Queries) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -38,9 +39,14 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-previous-release-ref @@ -87,6 +93,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -135,9 +142,14 @@ jobs: # Checkout to the last release of Vitess - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml new file mode 100644 index 00000000000..1c5659b4096 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml @@ -0,0 +1,216 @@ +name: Query Serving (Queries - 2) - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Queries - 2)') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + timeout-minutes: 60 + name: Run Upgrade Downgrade Test - Query Serving (Queries - 2) + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: 'false' + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_queries.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.23.4 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + timeout-minutes: 10 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Build current commit's binaries + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + rm -Rf bin/* + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n-1 and keep vttablet at version n + - name: Use last release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp -r /tmp/vitess-build-current/bin/* $PWD/bin/ + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n-1 and vttablet/vtctld at version n + - name: Run query serving tests (vtgate=N-1, vttablet=N, vtctld=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet/vtctld will be at version n-1 + - name: Use current version VTGate, and other version VTTablet/VTctld + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + + cp /tmp/vitess-build-other/bin/vtctld $PWD/bin + cp /tmp/vitess-build-other/bin/vtctldclient $PWD/bin + cp /tmp/vitess-build-other/bin/vtctl $PWD/bin + cp /tmp/vitess-build-other/bin/vtctlclient $PWD/bin + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet/vtctld at version n-1 + - name: Run query serving tests (vtgate=N, vttablet=N-1, vtctld=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml new file mode 100644 index 00000000000..bfd2d9021ae --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml @@ -0,0 +1,214 @@ +name: Query Serving (Queries - 2) Next Release - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Queries - 2) Next Release') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + timeout-minutes: 60 + name: Run Upgrade Downgrade Test - Query Serving (Queries - 2) Next Release + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: 'false' + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + timeout-minutes: 10 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n+1 and keep vttablet at version n + - name: Use next release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n+1 and vttablet at version n + - name: Run query serving tests (vtgate=N+1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 + - name: Use current version VTGate, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet at version n+1 + - name: Run query serving tests (vtgate=N, vttablet=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml index efc4b1033ab..bb328b9d78d 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Query Serving (Queries) Next Release runs-on: gh-hosted-runners-16cores-1-24.04 @@ -27,9 +28,14 @@ jobs: fi - name: Check out commit's code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-next-release-ref @@ -88,6 +94,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -136,9 +143,14 @@ jobs: # Checkout to the next release of Vitess - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -155,6 +167,30 @@ jobs: cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* +<<<<<<< HEAD +======= + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Convert ErrorContains checks to Error checks if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml index c6d61b14c87..a9044cc5ba4 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Query Serving (Schema) runs-on: gh-hosted-runners-16cores-1-24.04 @@ -38,9 +39,14 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-previous-release-ref @@ -87,6 +93,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -119,9 +126,14 @@ jobs: # Checkout to the last release of Vitess - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -141,7 +153,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml index 612d9a15d6a..48944be3299 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Query Serving (Schema) Next Release runs-on: gh-hosted-runners-16cores-1-24.04 @@ -27,9 +28,14 @@ jobs: fi - name: Check out commit's code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-next-release-ref @@ -88,6 +94,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -120,9 +127,14 @@ jobs: # Checkout to the next release of Vitess - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -142,7 +154,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml index fc481501152..7c7de1cd0ce 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Reparent New Vtctl runs-on: gh-hosted-runners-16cores-1-24.04 @@ -27,9 +28,14 @@ jobs: fi - name: Check out commit's code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-next-release-ref @@ -88,6 +94,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -120,9 +127,14 @@ jobs: # Checkout to the next release of Vitess - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -142,7 +154,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml index 60c14841c4d..e43241dbba3 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Reparent New VTTablet runs-on: gh-hosted-runners-16cores-1-24.04 @@ -27,9 +28,14 @@ jobs: fi - name: Check out commit's code +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-next-release-ref @@ -88,6 +94,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -127,9 +134,14 @@ jobs: # Checkout to the next release of Vitess - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -149,7 +161,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml index 5df9112b2dc..0cf4119c337 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Reparent Old Vtctl runs-on: gh-hosted-runners-16cores-1-24.04 @@ -38,9 +39,14 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-previous-release-ref @@ -87,6 +93,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -119,9 +126,14 @@ jobs: # Checkout to the last release of Vitess - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -141,7 +153,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml index b0b5da9a378..6f8ddb646b3 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -15,6 +15,7 @@ permissions: read-all jobs: upgrade_downgrade_test: + timeout-minutes: 60 name: Run Upgrade Downgrade Test - Reparent Old VTTablet runs-on: gh-hosted-runners-16cores-1-24.04 @@ -38,9 +39,14 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: fetch-depth: 0 + persist-credentials: 'false' - name: Set output with latest release branch id: output-previous-release-ref @@ -87,6 +93,7 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - name: Get base dependencies + timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -119,9 +126,14 @@ jobs: # Checkout to the last release of Vitess - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) with: ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -141,7 +153,13 @@ jobs: # Checkout to this build's commit - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Get dependencies for this commit if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml new file mode 100644 index 00000000000..f694624d951 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -0,0 +1,169 @@ +name: Semi Sync Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Semi Sync Upgrade Downgrade Testing') + cancel-in-progress: true + +permissions: read-all + +jobs: + upgrade_downgrade_test_e2e: + timeout-minutes: 60 + name: Run Semi Sync Upgrade Downgrade Test + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: 'false' + + - name: Set output with latest release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-previous-release-ref + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_semi_sync.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.23.4 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + timeout-minutes: 10 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + persist-credentials: 'false' + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Copy last releases vttablet + - name: Copy last release's VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttabletold + vttabletold --version + + - name: Run semi sync tests + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + go test -v -count=1 -run="" ./go/test/endtoend/reparent/semisync -alsologtostderr \ No newline at end of file diff --git a/.github/workflows/vitess_tester_vtgate.yml b/.github/workflows/vitess_tester_vtgate.yml index c970fdad744..dc8da12e73c 100644 --- a/.github/workflows/vitess_tester_vtgate.yml +++ b/.github/workflows/vitess_tester_vtgate.yml @@ -45,7 +45,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/vtadmin_web_build.yml b/.github/workflows/vtadmin_web_build.yml index 9b9ca3771ea..fc34221db6c 100644 --- a/.github/workflows/vtadmin_web_build.yml +++ b/.github/workflows/vtadmin_web_build.yml @@ -35,8 +35,14 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT +<<<<<<< HEAD - uses: actions/checkout@v4 +======= + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) if: steps.skip-workflow.outputs.skip-workflow == 'false' + with: + persist-credentials: 'false' - uses: actions/setup-node@v4 if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/vtadmin_web_lint.yml b/.github/workflows/vtadmin_web_lint.yml index bd0ce41c237..8fcd7c1ef36 100644 --- a/.github/workflows/vtadmin_web_lint.yml +++ b/.github/workflows/vtadmin_web_lint.yml @@ -35,8 +35,14 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT +<<<<<<< HEAD - uses: actions/checkout@v4 +======= + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) if: steps.skip-workflow.outputs.skip-workflow == 'false' + with: + persist-credentials: 'false' - uses: actions/setup-node@v4 if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/vtadmin_web_unit_tests.yml b/.github/workflows/vtadmin_web_unit_tests.yml index 3d6e7d60a8a..7170ce4a0cc 100644 --- a/.github/workflows/vtadmin_web_unit_tests.yml +++ b/.github/workflows/vtadmin_web_unit_tests.yml @@ -35,8 +35,14 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT +<<<<<<< HEAD - uses: actions/checkout@v4 +======= + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) if: steps.skip-workflow.outputs.skip-workflow == 'false' + with: + persist-credentials: 'false' - uses: actions/setup-node@v4 if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml new file mode 100644 index 00000000000..0d27d43b959 --- /dev/null +++ b/.github/workflows/vtop_example.yml @@ -0,0 +1,99 @@ +name: vtop_example +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example') + cancel-in-progress: true + +jobs: + build: + name: VTop Example + runs-on: self-hosted + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + - 'test/**' + - '.github/workflows/vtop_example.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Install everything we need, and configure + sudo apt-get install -y eatmydata make + go mod download + + # needed for vtctldclient + - name: Build vitess + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make build + + - name: Install kubectl & kind + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make install_kubectl_kind + + - name: vtop_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 60 + run: | + source build.env + eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example \ No newline at end of file diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 8e1369b7099..4e58b96c24e 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -13,6 +13,7 @@ env: jobs: build: + timeout-minutes: 60 name: Run endtoend tests on {{.Name}} runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}} @@ -43,7 +44,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -92,6 +99,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 run: | {{if .InstallXtraBackup}} diff --git a/test/templates/cluster_endtoend_test_docker.tpl b/test/templates/cluster_endtoend_test_docker.tpl index a87bdaee68a..12285d548e3 100644 --- a/test/templates/cluster_endtoend_test_docker.tpl +++ b/test/templates/cluster_endtoend_test_docker.tpl @@ -28,7 +28,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl index 022af234676..0eddd73279f 100644 --- a/test/templates/cluster_endtoend_test_mysql57.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -48,7 +48,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/test/templates/cluster_vitess_tester.tpl b/test/templates/cluster_vitess_tester.tpl index 5d4257e1069..82087e360a2 100644 --- a/test/templates/cluster_vitess_tester.tpl +++ b/test/templates/cluster_vitess_tester.tpl @@ -43,7 +43,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index cf8c9470b6a..1d5f5d41203 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -43,7 +43,13 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' +<<<<<<< HEAD uses: actions/checkout@v4 +======= + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' +>>>>>>> cbf146b5eb (Security improvements to GitHub Actions (#17520)) - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false'