Adding some debugging output & removing the duplicate line that was b… #670
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-deploy-pudl | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "v20*" | |
| branches: | |
| - try-google-batch | |
| schedule: | |
| # 6am UTC daily (11pm PDT, 2am EDT) | |
| # But only if there are changes since the last nightly build. | |
| - cron: "0 6 * * *" | |
| env: | |
| GCP_BILLING_PROJECT: ${{ secrets.GCP_BILLING_PROJECT }} | |
| GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule | |
| GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }} | |
| GCS_OUTPUT_BUCKET: gs://builds.catalyst.coop | |
| BATCH_JOB_JSON: batch_job.json | |
| jobs: | |
| build_and_deploy_pudl: | |
| name: Build Docker image, push to Docker Hub and deploy to a GCE VM | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| id-token: write | |
| steps: | |
| - name: Use pudl-deployment-dev vm if running on a schedule | |
| if: ${{ (github.event_name == 'schedule') }} | |
| run: | | |
| echo "This action was triggered by a schedule." | |
| echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Skip the build if no changes since the last successful nightly build. | |
| if: ${{ (github.event_name == 'schedule') }} | |
| run: | | |
| CURRENT_COMMIT=$(git rev-parse HEAD) | |
| NIGHTLY_COMMIT=$(git rev-parse origin/nightly) | |
| if [[ "$CURRENT_COMMIT" == "$NIGHTLY_COMMIT" ]]; then | |
| echo "::notice::No changes since last successful nightly build. Skipping." | |
| echo "SKIP_BUILD=true" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| - name: Set action environment variables | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| run: | | |
| echo "NIGHTLY_TAG=nightly-$(date +%Y-%m-%d)" >> $GITHUB_ENV | |
| echo "BUILD_ID=$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)-${{ github.ref_name }}" >> $GITHUB_ENV | |
| - name: Show freshly set envvars | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| run: | | |
| echo "GCE_INSTANCE: $GCE_INSTANCE" | |
| echo "NIGHTLY_TAG: $NIGHTLY_TAG" | |
| echo "BUILD_ID: $BUILD_ID" | |
| - name: Tag nightly build | |
| if: ${{ (github.event_name == 'schedule') && (env.SKIP_BUILD != 'true') }} | |
| run: | | |
| git config user.email "[email protected]" | |
| git config user.name "pudlbot" | |
| git tag -a -m "$NIGHTLY_TAG" $NIGHTLY_TAG ${{ github.ref_name }} | |
| git push origin $NIGHTLY_TAG | |
| - name: Docker Metadata | |
| id: docker_metadata | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: catalystcoop/pudl-etl | |
| flavor: | | |
| latest=auto | |
| tags: | | |
| type=raw,value=${{ github.ref_name }} | |
| type=ref,event=tag | |
| - name: Set up Docker Buildx | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| if: ${{ (github.event_name != 'pull_request') && (env.SKIP_BUILD != 'true') }} | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build image and push to Docker Hub | |
| id: docker-build | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.docker_metadata.outputs.tags }} | |
| labels: ${{ steps.docker_metadata.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: inspect docker manifest right after pushing | |
| continue-on-error: true | |
| run: docker inspect docker.io/catalystcoop/pudl-etl:${{ env.BUILD_REF}} | |
| - id: "auth" | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" | |
| service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com" | |
| # Setup gcloud CLI | |
| - name: Set up Cloud SDK | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: inspect docker tag manifest right before submitting Batch job | |
| continue-on-error: true | |
| run: docker inspect docker.io/catalystcoop/pudl-etl:${{ env.BUILD_REF}} | |
| - name: inspect docker hash manifest right before submitting Batch job | |
| continue-on-error: true | |
| run: docker inspect docker.io/catalystcoop/pudl-etl:${{ steps.docker-build.outputs.digest }} | |
| # Deploy PUDL image to GCE | |
| - name: Make GCP Batch config file | |
| if: ${{ env.SKIP_BUILD != 'true' }} | |
| env: | |
| PUDL_GCS_OUTPUT: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }} | |
| run: |- | |
| ./devtools/generate_batch_config.py \ | |
| --output ${{ env.BATCH_JOB_JSON }} \ | |
| --container-image "docker.io/catalystcoop/pudl-etl@${{ steps.docker-build.outputs.digest }}" \ | |
| --container-command "micromamba" \ | |
| --container-arg="run" \ | |
| --container-arg="--prefix" \ | |
| --container-arg="/home/mambauser/env" \ | |
| --container-arg="--attach" \ | |
| --container-arg='' \ | |
| --container-arg="bash" \ | |
| --container-arg="./docker/gcp_pudl_etl.sh" \ | |
| --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \ | |
| --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| --container-env BUILD_ID=${{ env.BUILD_ID }} \ | |
| --container-env BUILD_REF=${{ env.BUILD_REF }} \ | |
| --container-env DAGSTER_PG_DB="dagster" \ | |
| --container-env DAGSTER_PG_HOST="127.0.0.1" \ | |
| --container-env DAGSTER_PG_PASSWORD="dagster_password" \ | |
| --container-env DAGSTER_PG_USERNAME="dagster" \ | |
| --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \ | |
| --container-env GCE_INSTANCE=${{ env.GCE_INSTANCE }} \ | |
| --container-env GCE_INSTANCE_ZONE=${{ env.GCE_INSTANCE_ZONE }} \ | |
| --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \ | |
| --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \ | |
| --container-env NIGHTLY_TAG=${{ env.NIGHTLY_TAG }} \ | |
| --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \ | |
| --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_GCS_OUTPUT }} \ | |
| --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \ | |
| --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \ | |
| --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \ | |
| # Start the batch job | |
| - name: Kick off batch job | |
| run: gcloud batch jobs submit --job-prefix run-etl-test --config ${{ env.BATCH_JOB_JSON }} --location us-west1 | |
| - name: Post to a pudl-deployments channel | |
| id: slack | |
| uses: slackapi/slack-github-action@v1 | |
| with: | |
| channel-id: "C03FHB9N0PQ" | |
| slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_REF }}" | |
| env: | |
| channel-id: "C03FHB9N0PQ" | |
| SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} |