Improve nightly build and deployment logic

.github/release.yml

@@ -0,0 +1,17 @@
+---
+changelog:
+  exclude:
+    authors:
+      - dependabot
+      - pre-commit-ci
+      - pudlbot
+    labels:
+      - conda-lock
+      - dependencies
+  categories:
+    - title: New Data
+      labels:
+        - new-data
+    - title: Other Changes
+      labels:
+        - "*"

.github/workflows/build-deploy-pudl.yml

@@ -26,12 +26,22 @@ jobs:
         if: ${{ (github.event_name == 'schedule') }}
         run: |
           echo "This action was triggered by a schedule." && echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV && echo "GITHUB_REF=dev" >> $GITHUB_ENV
+          echo "NIGHTLY_TAG=nightly-$(date +%Y-%m-%d)" >> $GITHUB_ENV
+          echo "NIGHTLY_TAG: $NIGHTLY_TAG"
 
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
           ref: ${{ env.GITHUB_REF }}
 
+      - name: Tag nightly build
+        if: ${{ (github.event_name == 'schedule') }}
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "PudlBot"
+          git tag -a -m "$NIGHTLY_TAG" $NIGHTLY_TAG $GITHUB_REF
+          git push origin $NIGHTLY_TAG
+
       - name: Get HEAD of the branch (main or dev)
         run: |
           echo "ACTION_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
@@ -45,7 +55,7 @@ jobs:
 
       - name: Docker Metadata
         id: docker_metadata
-        uses: docker/metadata-action@v5.3.0
+        uses: docker/metadata-action@v5
         with:
           images: catalystcoop/pudl-etl
           flavor: |
@@ -55,17 +65,17 @@ jobs:
             type=ref,event=tag
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3.0.0
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build image and push to Docker Hub
-        uses: docker/build-push-action@v5.1.0
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker/Dockerfile
@@ -114,6 +124,7 @@ jobs:
             --container-env-file="./docker/.env" \
             --container-env ACTION_SHA=$ACTION_SHA \
             --container-env GITHUB_REF=${{ env.GITHUB_REF }} \
+            --container-env NIGHTLY_TAG=${{ env.NIGHTLY_TAG }} \
             --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \
             --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \
             --container-env GCE_INSTANCE=${{ env.GCE_INSTANCE }} \
@@ -127,8 +138,9 @@ jobs:
             --container-env DAGSTER_PG_HOST="104.154.182.24" \
             --container-env DAGSTER_PG_DB="dagster-storage" \
             --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \
+            --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \
             --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \
-            --container-env PUDL_SETTINGS_YML="/home/mambauser/src/pudl/package_data/settings/etl_full.yml" \
+            --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \
             --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_OUTPUT_PATH }}
 
       # Start the VM
@@ -137,7 +149,7 @@ jobs:
 
       - name: Post to a pudl-deployments channel
         id: slack
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@v1
         with:
           channel-id: "C03FHB9N0PQ"
           slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.GCS_OUTPUT_BUCKET }}/${{ env.RUN_TIMESTAMP}}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}"

.github/workflows/docker-build-test.yml

@@ -17,17 +17,17 @@ jobs:
 
       - name: Docker Metadata
         id: docker_metadata
-        uses: docker/metadata-action@v5.3.0
+        uses: docker/metadata-action@v5
         with:
           images: catalystcoop/pudl-etl
           flavor: |
             latest=auto
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
+        uses: docker/setup-buildx-action@v3
 
       - name: Build image but do not push to Docker Hub
-        uses: docker/build-push-action@v5.1.0
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker/Dockerfile

.github/workflows/release.yml

@@ -83,7 +83,7 @@ jobs:
           name: python-package-distributions
           path: dist/
       - name: Sign dist with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v2.1.0
+        uses: sigstore/gh-action-sigstore-python@v2
         with:
           inputs: ./dist/*.tar.gz ./dist/*.whl
       - name: Upload artifact signatures to GitHub Release
@@ -95,6 +95,13 @@ jobs:
           --discussion-category Announcements
           --generate-notes
           --repo '${{ github.repository }}'
+      - name: Tag the current stable commit
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "pudlbot"
+          git checkout stable
+          git merge --ff-only ${{ github.ref_name }}
+          git push
 
   notify-slack:
     runs-on: ubuntu-latest

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM mambaorg/micromamba:1.5.3
+FROM mambaorg/micromamba:1.5.5
 
 USER root
 
@@ -30,28 +30,27 @@ ENV PUDL_INPUT=${CONTAINER_PUDL_WORKSPACE}/input
 ENV PUDL_OUTPUT=${CONTAINER_PUDL_WORKSPACE}/output
 ENV DAGSTER_HOME=${CONTAINER_PUDL_WORKSPACE}/dagster_home
 
-RUN mkdir -p ${PUDL_INPUT} ${PUDL_OUTPUT} ${DAGSTER_HOME}
+RUN mkdir -p ${PUDL_INPUT} ${PUDL_OUTPUT} ${DAGSTER_HOME} ${PUDL_REPO}
 
 # Copy dagster configuration file
 COPY docker/dagster.yaml ${DAGSTER_HOME}/dagster.yaml
 
+# Copy the cloned pudl repository into the container
+# This includes the .git directory, so it is a whole repo
+COPY --chown=${MAMBA_USER}:${MAMBA_USER} . ${PUDL_REPO}
+
 # Create a conda environment based on the specification in the repo
-COPY environments/conda-lock.yml environments/conda-lock.yml
-RUN micromamba create --prefix ${CONDA_PREFIX} --yes --file environments/conda-lock.yml && \
+RUN micromamba create --prefix ${CONDA_PREFIX} --yes --file ${PUDL_REPO}/environments/conda-lock.yml && \
     micromamba clean -afy
-# Copy the cloned pudl repository into the user's home directory
-COPY --chown=${MAMBA_USER}:${MAMBA_USER} . ${CONTAINER_HOME}
 
 # TODO(rousik): The following is a workaround for sudden breakage where conda
 # can't find libraries contained within the environment. It's unclear why!
 ENV LD_LIBRARY_PATH=${CONDA_PREFIX}/lib
-# We need information from .git to get version with setuptools_scm so we mount that
-# directory without copying it into the image.
-RUN --mount=type=bind,source=.git,target=${PUDL_REPO}/.git \
-    ${CONDA_RUN} pip install --no-cache-dir --no-deps --editable .
+RUN ${CONDA_RUN} pip install --no-cache-dir --no-deps --editable ${PUDL_REPO}
 
 # Install awscli2
 # Change back to root because the install script needs access to /usr/local/aws-cli
+# curl commands run within conda environment because curl is installed by conda.
 USER root
 RUN ${CONDA_RUN} bash -c 'curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install'
 USER $MAMBA_USER
@@ -61,6 +60,6 @@ USER $MAMBA_USER
 RUN ${CONDA_RUN} bash -c 'curl -L https://fly.io/install.sh | sh'
 ENV PATH="${CONTAINER_HOME}/.fly/bin:$PATH"
 
-
+WORKDIR ${PUDL_REPO}
 # Run the unit tests:
 CMD ["micromamba", "run", "--prefix", "${CONDA_PREFIX}", "--attach", "''", "pytest", "test/unit"]

docker/gcp_pudl_etl.sh

@@ -48,16 +48,12 @@ function run_pudl_etl() {
 }
 
 function shutdown_vm() {
-    # Copy the outputs to the GCS bucket
-
     upload_file_to_slack $LOGFILE "pudl_etl logs for $ACTION_SHA-$GITHUB_REF:"
-
+    # Shut down the vm instance when the etl is done.
     echo "Shutting down VM."
-    # # Shut down the vm instance when the etl is done.
-    ACCESS_TOKEN=`curl \
+    ACCESS_TOKEN=$(curl \
         "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \
-        -H "Metadata-Flavor: Google" | jq -r '.access_token'`
-
+        -H "Metadata-Flavor: Google" | jq -r '.access_token')
     curl -X POST -H "Content-Length: 0" -H "Authorization: Bearer ${ACCESS_TOKEN}" https://compute.googleapis.com/compute/v1/projects/catalyst-cooperative-pudl/zones/$GCE_INSTANCE_ZONE/instances/$GCE_INSTANCE/stop
 }
 
@@ -68,19 +64,25 @@ function copy_outputs_to_gcs() {
 }
 
 function copy_outputs_to_distribution_bucket() {
-    echo "Copying outputs to GCP distribution bucket"
-    gsutil -m -u $GCP_BILLING_PROJECT cp -r "$PUDL_OUTPUT/*" "gs://pudl.catalyst.coop/$GITHUB_REF"
-
-    echo "Copying outputs to AWS distribution bucket"
-    aws s3 cp "$PUDL_OUTPUT/" "s3://pudl.catalyst.coop/$GITHUB_REF" --recursive
+    # Only attempt to update outputs if we have a real value of GITHUB_REF
+    if [ -n "$GITHUB_REF" ]; then
+        echo "Removing old $GITHUB_REF outputs from GCP distributon bucket."
+        gsutil -m -u $GCP_BILLING_PROJECT rm -r "gs://pudl.catalyst.coop/$GITHUB_REF"
+        echo "Copying outputs to GCP distribution bucket"
+        gsutil -m -u $GCP_BILLING_PROJECT cp -r "$PUDL_OUTPUT/*" "gs://pudl.catalyst.coop/$GITHUB_REF"
+
+        echo "Removing old $GITHUB_REF outputs from AWS distributon bucket."
+        aws s3 rm "s3://pudl.catalyst.coop/$GITHUB_REF" --recursive
+        echo "Copying outputs to AWS distribution bucket"
+        aws s3 cp "$PUDL_OUTPUT/" "s3://pudl.catalyst.coop/$GITHUB_REF" --recursive
+    fi
 }
 
 function zenodo_data_release() {
     echo "Creating a new PUDL data release on Zenodo."
     ~/devtools/zenodo/zenodo_data_release.py --publish --env sandbox --source-dir $PUDL_OUTPUT
 }
 
-
 function notify_slack() {
     # Notify pudl-builds slack channel of deployment status
     if [ $1 = "success" ]; then
@@ -97,6 +99,19 @@ function notify_slack() {
     send_slack_msg "$message"
 }
 
+if [ "$GITHUB_ACTION_TRIGGER" = "workflow_dispatch" ]; then
+    echo "Deployed via workflow_dispatch, testing git authentication!"
+    # Remove the read-only authentication header
+    git config --unset http.https://github.com/.extraheader
+    git config user.email "[email protected]"
+    git config user.name "pudlbot"
+    git remote set-url origin "https://pudlbot:[email protected]/catalyst-cooperative/pudl.git"
+    git config -l
+    git tag -a -m "Nightly test tag" nightly-tag-test
+    git push origin nightly-tag-test
+    shutdown_vm
+fi
+
 # # Run ETL. Copy outputs to GCS and shutdown VM if ETL succeeds or fails
 # 2>&1 redirects stderr to stdout.
 run_pudl_etl 2>&1 | tee $LOGFILE
@@ -107,6 +122,15 @@ copy_outputs_to_gcs
 
 # if pipeline is successful, distribute + publish datasette
 if [[ $ETL_SUCCESS == 0 ]]; then
+    if [ $GITHUB_ACTION_TRIGGER = "schedule" ]; then
+        # Update the nightly branch to point at newly successful nightly build tag
+        git config user.email "[email protected]"
+        git config user.name "pudlbot"
+        git remote set-url origin https://pudlbot:[email protected]/catalyst-cooperative/pudl.git
+        git checkout nightly
+        git merge --ff-only $NIGHTLY_TAG
+        git push
+    fi
     # Deploy the updated data to datasette
     if [ $GITHUB_REF = "dev" ]; then
         python ~/devtools/datasette/publish.py 2>&1 | tee -a $LOGFILE