From 06799bcaa218ef761fef901044bc92ca27401fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 1 Sep 2024 19:38:11 +0200 Subject: [PATCH] Use the Trino run script in CI workflows --- .github/workflows/optimize.yml | 25 ++--------------------- .github/workflows/reports.yml | 23 ++------------------- .github/workflows/sync.yml | 37 +--------------------------------- .gitignore | 2 -- 4 files changed, 5 insertions(+), 82 deletions(-) diff --git a/.github/workflows/optimize.yml b/.github/workflows/optimize.yml index d17a5234..d6aa0a5c 100644 --- a/.github/workflows/optimize.yml +++ b/.github/workflows/optimize.yml @@ -13,35 +13,14 @@ jobs: sync: runs-on: ubuntu-latest env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} - TRINO_VERSION: 445 steps: - uses: actions/checkout@v4 - name: Start Trino - run: | - CONTAINER_ID=$(docker run \ - -v $(pwd)/catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \ - -v $(pwd)/sql:/sql \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION \ - -p 8080:8080 \ - --name trino \ - -d \ - trinodb/trino:$TRINO_VERSION) - SERVER_IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID") - echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV - echo "SERVER_IP=$SERVER_IP" >> $GITHUB_ENV - i=0 - until docker inspect "${CONTAINER_ID}" --format "{{json .State.Health.Status }}" | grep -q '"healthy"'; do - if [[ $((i++)) -ge 10 ]]; then - echo "🚨 Too many retries waiting for Trino to start" - exit 1 - fi - sleep 10 - done + run: ./bin/run-trino.sh - name: Optimize tables run: | docker exec \ diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 36451a19..bde5ee48 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -16,37 +16,18 @@ jobs: run: runs-on: ubuntu-latest env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} - TRINO_VERSION: 445 - TRINO_GIT_VERSION: "0.74" steps: - uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt update sudo pip install ansi2html - curl -fLOsS https://github.com/nineinchnick/trino-git/releases/download/v$TRINO_GIT_VERSION/trino-git-$TRINO_GIT_VERSION.zip - unzip trino-git-$TRINO_GIT_VERSION.zip - name: Start Trino - run: | - CONTAINER_ID=$(docker run \ - -v $(pwd)/trino-git-$TRINO_GIT_VERSION:/usr/lib/trino/plugin/git \ - -v $(pwd)/catalog/git.properties:/etc/trino/catalog/git.properties \ - -v $(pwd)/catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \ - -v $(pwd)/hive-cache:/opt/hive-cache \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION \ - -p 8080:8080 \ - --name trino \ - -d \ - trinodb/trino:$TRINO_VERSION) - SERVER_IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID") - echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV - echo "SERVER_IP=$SERVER_IP" >> $GITHUB_ENV - until docker logs trino 2>&1 | grep --quiet --fixed-strings --max-count=1 "SERVER STARTED"; do sleep 1 ; done + run: ./bin/run-trino.sh - name: Execute queries run: | ./bin/reports.sh reports/ci-cd/index.md "Trino CI/CD Reports" sql/ci-cd/{health,success-ratio-per-day,runs-queue-time-per-day,runs-duration-per-day,runs-job-cumulative-duration-per-day,jobs-duration}.sql diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index f75d578c..aeca9934 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -17,7 +17,6 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} - TRINO_VERSION: 445 TRINO_REST_VERSION: "0.144" GITHUB_OWNER: trinodb GITHUB_REPO: trino @@ -28,45 +27,11 @@ jobs: SYNC_TABLES: runs,jobs,steps,check_suites,check_runs,check_run_annotations,pulls,pull_commits,pull_stats,review_comments,reviews,issues,issue_comments,commits,teams,members steps: - uses: actions/checkout@v4 - - name: Download trino-rest - run: | - curl -fLOsS https://github.com/nineinchnick/trino-rest/releases/download/v$TRINO_REST_VERSION/trino-rest-github-$TRINO_REST_VERSION.zip - unzip trino-rest-github-$TRINO_REST_VERSION.zip - name: Start Trino run: | - cat <config.properties - coordinator=true - node-scheduler.include-coordinator=true - http-server.http.port=8080 - discovery.uri=http://localhost:8080 - query.max-memory-per-node=4086929818B - EOF - CONTAINER_ID=$(docker run \ - -v $(pwd)/config.properties:/etc/trino/config.properties \ - -v $(pwd)/trino-rest-github-$TRINO_REST_VERSION:/usr/lib/trino/plugin/github \ - -v $(pwd)/catalog/github.properties:/etc/trino/catalog/github.properties \ - -v $(pwd)/catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \ - -v $(pwd)/hive-cache:/opt/hive-cache \ - -v $(pwd)/sql:/sql \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION \ - -e GITHUB_TOKEN \ - -p 8080:8080 \ - --name trino \ - -d \ - trinodb/trino:$TRINO_VERSION) + ./bin/run-trino.sh SERVER_IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID") - echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV echo "SERVER_IP=$SERVER_IP" >> $GITHUB_ENV - i=0 - until docker inspect "${CONTAINER_ID}" --format "{{json .State.Health.Status }}" | grep -q '"healthy"'; do - if [[ $((i++)) -ge 10 ]]; then - echo "🚨 Too many retries waiting for Trino to start" - exit 1 - fi - sleep 10 - done - name: Run Sync run: | docker run \ diff --git a/.gitignore b/.gitignore index a3ab9fb5..73677282 100644 --- a/.gitignore +++ b/.gitignore @@ -107,8 +107,6 @@ dist .idea # custom Trino connectors -/trino-rest-github-* -/trino-git-* /bin/trino-rest-github-* /bin/trino-git-* /bin/config.properties