Merge pull request #3070 from catalyst-cooperative/nightly-build-2023…

…-11-21

Merge dev into main for 2023-11-21

.gitattributes

@@ -1,6 +1,7 @@
 *.ipynb linguist-detectable=false
 *.html linguist-detectable=false
 eia861-transform.ipynb merge=ours
+environments/conda-*lock.yml merge=ours
 *.csv text
 *.py text
 *.json text

.github/workflows/update-conda-lockfile.yml

@@ -49,7 +49,7 @@ jobs:
           environment-name: conda-lock
           create-args: >-
             python=3.11
-            conda-lock
+            conda-lock>=2.5.1
             prettier
       - name: Run conda-lock to recreate lockfile from scratch
         run: |

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
   # Formatters: hooks that re-write Python & documentation files
   ####################################################################################
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.5
+    rev: v0.1.6
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

docker/gcp_pudl_etl.sh

@@ -91,23 +91,27 @@ function notify_slack() {
 # 2>&1 redirects stderr to stdout.
 run_pudl_etl 2>&1 | tee $LOGFILE
 
+ETL_SUCCESS=${PIPESTATUS[0]}
+
 # if pipeline is successful, distribute + publish datasette
-if [[ ${PIPESTATUS[0]} == 0 ]]; then
+if [[ $ETL_SUCCESS == 0 ]]; then
     # Dump outputs to s3 bucket if branch is dev or build was triggered by a tag
     if [ $GITHUB_ACTION_TRIGGER = "push" ] || [ $GITHUB_REF = "dev" ]; then
         copy_outputs_to_distribution_bucket
+        ETL_SUCCESS=${PIPESTATUS[0]}
     fi
 
     # Deploy the updated data to datasette
     if [ $GITHUB_REF = "dev" ]; then
         python ~/devtools/datasette/publish.py 2>&1 | tee -a $LOGFILE
+        ETL_SUCCESS=${PIPESTATUS[0]}
     fi
 fi
 
 # Notify slack about entire pipeline's success or failure;
 # PIPESTATUS[0] either refers to the failed ETL run or the last distribution
 # task that was run above
-if [[ ${PIPESTATUS[0]} == 0 ]]; then
+if [[ $ETL_SUCCESS == 0 ]]; then
     notify_slack "success"
 else
     notify_slack "failure"

environments/conda-linux-64.lock.yml

@@ -1,13 +1,13 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 62bcd0fc7d50df96982d048631ad45bff7910db618111621e212e7f3c28a44bc
+# input_hash: 948078f7f90587f6b91354dbc73ccdcb6abab18c115f55a56d7f4eba9cd0cd6d
 
 channels:
   - conda-forge
   - defaults
 dependencies:
   - _libgcc_mutex=0.1=conda_forge
-  - ca-certificates=2023.7.22=hbcca054_0
+  - ca-certificates=2023.11.17=hbcca054_0
   - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
   - font-ttf-inconsolata=3.000=h77eed37_0
   - font-ttf-source-code-pro=2.038=h77eed37_0
@@ -26,7 +26,7 @@ dependencies:
   - libgcc-ng=13.2.0=h807b86a_3
   - aws-c-common=0.9.8=hd590300_0
   - bzip2=1.0.8=hd590300_5
-  - c-ares=1.21.0=hd590300_0
+  - c-ares=1.22.1=hd590300_0
   - fribidi=1.0.10=h36c2ea0_0
   - geos=3.12.0=h59595ed_0
   - gettext=0.21.1=h27087fc_0
@@ -101,7 +101,7 @@ dependencies:
   - libsqlite=3.44.0=h2797004_0
   - libssh2=1.11.0=h0841786_0
   - libxcb=1.15=h0b41bf4_0
-  - libxml2=2.11.5=h232c23b_1
+  - libxml2=2.11.6=h232c23b_0
   - libzip=1.10.1=h2629f0a_3
   - pcre2=10.42=hcad00b1_0
   - readline=8.2=h8228510_1
@@ -125,7 +125,7 @@ dependencies:
   - libtiff=4.6.0=ha9c0a0a_2
   - libxslt=1.1.37=h0054252_1
   - minizip=4.0.3=h0ab5242_0
-  - nodejs=20.8.1=h1990674_0
+  - nodejs=20.9.0=hb753e55_0
   - nss=3.94=h1d7d5a4_0
   - orc=1.9.0=h4b38347_4
   - pandoc=3.1.3=h32600fe_0
@@ -153,7 +153,7 @@ dependencies:
   - cachy=0.3.0=pyhd8ed1ab_1
   - catalystcoop.dbfread=3.0.0=py_0
   - cchardet=2.1.7=py311hb755f60_5
-  - certifi=2023.7.22=pyhd8ed1ab_0
+  - certifi=2023.11.17=pyhd8ed1ab_0
   - cfgv=3.3.1=pyhd8ed1ab_0
   - chardet=5.2.0=py311h38be061_1
   - charset-normalizer=3.3.2=pyhd8ed1ab_0
@@ -196,7 +196,7 @@ dependencies:
   - iniconfig=2.0.0=pyhd8ed1ab_0
   - itsdangerous=2.1.2=pyhd8ed1ab_0
   - jeepney=0.8.0=pyhd8ed1ab_0
-  - jellyfish=1.0.1=py311h46250e7_1
+  - jellyfish=1.0.3=py311h46250e7_0
   - jmespath=1.0.1=pyhd8ed1ab_0
   - json5=0.9.14=pyhd8ed1ab_0
   - jsonpointer=2.4=py311h38be061_3
@@ -211,7 +211,7 @@ dependencies:
   - llvmlite=0.41.1=py311ha6695c7_0
   - locket=1.0.0=pyhd8ed1ab_0
   - lxml=4.9.3=py311h1a07684_1
-  - marko=1.3.1=pyhd8ed1ab_0
+  - marko=2.0.2=pyhd8ed1ab_0
   - markupsafe=2.1.3=py311h459d7ec_1
   - mdurl=0.1.0=pyhd8ed1ab_0
   - mergedeep=1.3.4=pyhd8ed1ab_0
@@ -243,7 +243,7 @@ dependencies:
   - pure_eval=0.2.2=pyhd8ed1ab_0
   - pyasn1=0.5.0=pyhd8ed1ab_0
   - pycparser=2.21=pyhd8ed1ab_0
-  - pygments=2.16.1=pyhd8ed1ab_0
+  - pygments=2.17.1=pyhd8ed1ab_0
   - pyjwt=2.8.0=pyhd8ed1ab_0
   - pylev=1.4.0=pyhd8ed1ab_0
   - pyparsing=3.1.1=pyhd8ed1ab_0
@@ -262,10 +262,10 @@ dependencies:
   - regex=2023.10.3=py311h459d7ec_0
   - rfc3986=2.0.0=pyhd8ed1ab_0
   - rfc3986-validator=0.1.1=pyh9f0ad1d_0
-  - rpds-py=0.13.0=py311h46250e7_0
+  - rpds-py=0.13.1=py311h46250e7_0
   - rtree=1.1.0=py311h3bb2b0f_0
   - ruamel.yaml.clib=0.2.7=py311h459d7ec_2
-  - ruff=0.1.5=py311h7145743_0
+  - ruff=0.1.6=py311h7145743_0
   - send2trash=1.8.2=pyh41d4057_0
   - setuptools=68.2.2=pyhd8ed1ab_0
   - shellingham=1.5.4=pyhd8ed1ab_0
@@ -315,7 +315,7 @@ dependencies:
   - asgiref=3.7.2=pyhd8ed1ab_0
   - asttokens=2.4.1=pyhd8ed1ab_0
   - async-lru=2.0.4=pyhd8ed1ab_0
-  - aws-c-auth=0.7.6=h37ad1db_0
+  - aws-c-auth=0.7.7=h37ad1db_0
   - aws-c-mqtt=0.9.9=h1387108_0
   - babel=2.13.1=pyhd8ed1ab_0
   - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
@@ -333,15 +333,15 @@ dependencies:
   - coloredlogs=14.0=pyhd8ed1ab_3
   - comm=0.1.4=pyhd8ed1ab_0
   - coverage=7.3.2=py311h459d7ec_0
-  - fonttools=4.44.3=py311h459d7ec_0
+  - fonttools=4.45.0=py311h459d7ec_0
   - gitdb=4.0.11=pyhd8ed1ab_0
   - graphql-core=3.2.3=pyhd8ed1ab_0
   - grpcio=1.59.2=py311ha6695c7_0
   - h11=0.14.0=pyhd8ed1ab_0
   - h2=4.1.0=pyhd8ed1ab_0
   - hdf5=1.14.2=nompi_h4f84152_100
   - html5lib=1.1=pyh9f0ad1d_0
-  - hypothesis=6.89.0=pyha770c72_0
+  - hypothesis=6.90.0=pyha770c72_0
   - importlib-metadata=6.8.0=pyha770c72_0
   - importlib_resources=6.1.1=pyhd8ed1ab_0
   - isodate=0.6.1=pyhd8ed1ab_0
@@ -384,7 +384,6 @@ dependencies:
   - rfc3339-validator=0.1.4=pyhd8ed1ab_0
   - rsa=4.9=pyhd8ed1ab_0
   - ruamel.yaml=0.18.5=py311h459d7ec_0
-  - sqlalchemy=1.4.49=py311h459d7ec_1
   - terminado=0.18.0=pyh0d859eb_0
   - tinycss2=1.2.1=pyhd8ed1ab_0
   - tqdm=4.66.1=pyhd8ed1ab_0
@@ -400,8 +399,8 @@ dependencies:
   - argon2-cffi-bindings=21.2.0=py311h459d7ec_4
   - arrow=1.3.0=pyhd8ed1ab_0
   - async-timeout=4.0.3=pyhd8ed1ab_0
-  - aws-c-s3=0.3.24=h7630044_0
-  - botocore=1.32.2=pyhd8ed1ab_0
+  - aws-c-s3=0.3.24=hdb3bed3_1
+  - botocore=1.32.4=pyhd8ed1ab_0
   - branca=0.7.0=pyhd8ed1ab_1
   - croniter=2.0.1=pyhd8ed1ab_0
   - cryptography=41.0.5=py311h63ff55d_0
@@ -438,6 +437,7 @@ dependencies:
   - python-build=1.0.3=pyhd8ed1ab_0
   - requests=2.31.0=pyhd8ed1ab_0
   - rich=13.7.0=pyhd8ed1ab_0
+  - sqlalchemy=2.0.23=py311h459d7ec_0
   - stack_data=0.6.2=pyhd8ed1ab_0
   - starlette=0.32.0.post1=pyhd8ed1ab_0
   - tiledb=2.16.3=h8c794c1_3
@@ -449,7 +449,7 @@ dependencies:
   - alembic=1.12.1=pyhd8ed1ab_0
   - arelle-release=2.17.4=pyhd8ed1ab_0
   - argon2-cffi=23.1.0=pyhd8ed1ab_0
-  - aws-crt-cpp=0.24.7=h4712614_1
+  - aws-crt-cpp=0.24.7=hd0f6be0_2
   - black=23.10.1=py311h38be061_0
   - bottleneck=1.3.7=py311h1f0f07a_1
   - cachecontrol=0.13.1=pyhd8ed1ab_0
@@ -463,11 +463,11 @@ dependencies:
   - grpcio-status=1.59.2=pyhd8ed1ab_0
   - h3-py=3.7.6=py311hb755f60_1
   - httpx=0.25.1=pyhd8ed1ab_0
-  - identify=2.5.31=pyhd8ed1ab_0
+  - identify=2.5.32=pyhd8ed1ab_0
   - isoduration=20.11.0=pyhd8ed1ab_0
   - jsonschema=4.20.0=pyhd8ed1ab_0
   - jupyter_core=5.5.0=py311h38be061_0
-  - libgdal=3.8.0=h12dd931_4
+  - libgdal=3.8.0=h12dd931_5
   - numba=0.58.1=py311h96b013e_0
   - numexpr=2.8.7=py311h039bad6_104
   - oauthlib=3.2.2=pyhd8ed1ab_0
@@ -489,14 +489,14 @@ dependencies:
   - uvicorn-standard=0.24.0=h38be061_0
   - virtualenv=20.24.6=pyhd8ed1ab_0
   - aws-sdk-cpp=1.11.182=h8beafcf_7
-  - boto3=1.29.2=pyhd8ed1ab_0
+  - boto3=1.29.4=pyhd8ed1ab_0
   - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0
   - dagster=1.5.9=pyhd8ed1ab_0
   - datasette=0.64.4=pyhd8ed1ab_1
   - doc8=1.1.1=pyhd8ed1ab_0
   - email-validator=2.1.0.post1=pyhd8ed1ab_0
   - frictionless=4.40.8=pyh6c4a22f_0
-  - gdal=3.8.0=py311h815a124_4
+  - gdal=3.8.0=py311h815a124_5
   - geopandas-base=0.14.1=pyha770c72_0
   - google-auth=2.23.4=pyhca7485f_0
   - gql-with-requests=3.4.1=pyhd8ed1ab_0
@@ -505,7 +505,7 @@ dependencies:
   - jupyter_client=8.6.0=pyhd8ed1ab_0
   - keyring=24.3.0=py311h38be061_0
   - librsvg=2.56.3=h98fae49_0
-  - matplotlib-base=3.8.1=py311h54ef318_0
+  - matplotlib-base=3.8.2=py311h54ef318_0
   - nbformat=5.9.2=pyhd8ed1ab_0
   - pandera-core=0.17.2=pyhd8ed1ab_1
   - pre-commit=3.5.0=pyha770c72_0
@@ -514,13 +514,13 @@ dependencies:
   - scikit-learn=1.3.2=py311hc009520_1
   - timezonefinder=6.2.0=py311h459d7ec_2
   - catalystcoop.ferc_xbrl_extractor=1.2.1=pyhd8ed1ab_0
-  - conda-lock=2.4.2=pyhd8ed1ab_0
+  - conda-lock=2.5.1=pyhd8ed1ab_0
   - dagster-graphql=1.5.9=pyhd8ed1ab_0
   - dagster-postgres=0.21.9=pyhd8ed1ab_0
   - fiona=1.9.5=py311hf8e0aa6_1
   - google-api-core=2.14.0=pyhd8ed1ab_0
   - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0
-  - graphviz=8.1.0=h28d9a01_0
+  - graphviz=9.0.0=h28d9a01_0
   - ipython=8.17.2=pyh41d4057_0
   - jupyter_events=0.9.0=pyhd8ed1ab_0
   - libarrow=14.0.1=h4df1b6a_3_cpu
@@ -538,7 +538,7 @@ dependencies:
   - libarrow-gandiva=14.0.1=hacb8726_3_cpu
   - libparquet=14.0.1=h352af49_3_cpu
   - nbconvert-core=7.11.0=pyhd8ed1ab_0
-  - pygraphviz=1.11=py311h72a77b7_1
+  - pygraphviz=1.11=py311hbf5cbc9_2
   - tableschema=1.19.3=pyh9f0ad1d_0
   - datapackage=1.15.2=pyh44b312d_0
   - google-cloud-storage=2.13.0=pyhca7485f_0
@@ -551,11 +551,11 @@ dependencies:
   - gcsfs=2023.10.0=pyhd8ed1ab_0
   - jupyter-lsp=2.2.0=pyhd8ed1ab_0
   - jupyter-resource-usage=1.0.1=pyhd8ed1ab_0
-  - jupyterlab_server=2.25.1=pyhd8ed1ab_0
+  - jupyterlab_server=2.25.2=pyhd8ed1ab_0
   - libarrow-substrait=14.0.1=h61ff412_3_cpu
   - nbconvert=7.11.0=pyhd8ed1ab_0
   - notebook-shim=0.2.3=pyhd8ed1ab_0
-  - jupyterlab=4.0.8=pyhd8ed1ab_0
+  - jupyterlab=4.0.9=pyhd8ed1ab_0
   - pyarrow=14.0.1=py311h39c9aba_3_cpu
   - notebook=7.0.6=pyhd8ed1ab_0
   - jupyter=1.0.0=pyhd8ed1ab_10