diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml index 1b59220fd9..d9dded930a 100644 --- a/.github/workflows/build-deploy-pudl.yml +++ b/.github/workflows/build-deploy-pudl.yml @@ -133,7 +133,7 @@ jobs: - name: Make GCP Batch config file if: ${{ env.SKIP_BUILD != 'true' }} env: - PUDL_OUTPUT_PATH: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }} + PUDL_GCS_OUTPUT: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }} run: |- ./devtools/generate_batch_config.py \ --output ${{ env.BATCH_JOB_JSON }} \ @@ -162,8 +162,7 @@ jobs: --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \ --container-env NIGHTLY_TAG=${{ env.NIGHTLY_TAG }} \ --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \ - --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_GCS_OUTPUT }} - --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_OUTPUT_PATH }} \ + --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_GCS_OUTPUT }} \ --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \ --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \ --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \ diff --git a/docker/gcp_pudl_etl.sh b/docker/gcp_pudl_etl.sh index 2b0635a370..ffe906afd7 100644 --- a/docker/gcp_pudl_etl.sh +++ b/docker/gcp_pudl_etl.sh @@ -3,6 +3,7 @@ # This script won't work locally because it needs adequate GCP permissions. function send_slack_msg() { + echo "sending slack message" curl -X POST -H "Content-type: application/json" -H "Authorization: Bearer ${SLACK_TOKEN}" https://slack.com/api/chat.postMessage --data "{\"channel\": \"C03FHB9N0PQ\", \"text\": \"$1\"}" } @@ -13,15 +14,18 @@ function upload_file_to_slack() { function authenticate_gcp() { # Set the default gcloud project id so the zenodo-cache bucket # knows what project to bill for egress + echo "authenticating GCP" gcloud config set project "$GCP_BILLING_PROJECT" } function initialize_postgres() { + echo "initializing postgres" sudo -u postgres pg_ctl start -l logfile && \ sudo -u postgres psql -c "CREATE USER dagster WITH SUPERUSER PASSWORD 'dagster_password'" } function run_pudl_etl() { + echo "starting to run PUDL ETL" send_slack_msg ":large_yellow_circle: Deployment started for $BUILD_ID :floppy_disk:" initialize_postgres && \ authenticate_gcp && \ @@ -48,16 +52,6 @@ function run_pudl_etl() { && touch "$PUDL_OUTPUT/success" } -function shutdown_vm() { - upload_file_to_slack "$LOGFILE" "pudl_etl logs for $BUILD_ID:" - # Shut down the vm instance when the etl is done. - echo "Shutting down VM." - ACCESS_TOKEN=$(curl \ - "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \ - -H "Metadata-Flavor: Google" | jq -r '.access_token') - curl -X POST -H "Content-Length: 0" -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://compute.googleapis.com/compute/v1/projects/catalyst-cooperative-pudl/zones/$GCE_INSTANCE_ZONE/instances/$GCE_INSTANCE/stop" -} - function save_outputs_to_gcs() { echo "Copying outputs to GCP bucket $PUDL_GCS_OUTPUT" && \ gsutil -m cp -r "$PUDL_OUTPUT" "$PUDL_GCS_OUTPUT" && \ @@ -213,6 +207,4 @@ if [[ $ETL_SUCCESS == 0 && \ notify_slack "success" else notify_slack "failure" -fi - -shutdown_vm +fi \ No newline at end of file