Add new archiver for EPA eGRID

.github/workflows/run-archiver.yml

@@ -6,7 +6,7 @@ on:
     inputs:
       datasets:
         description: 'Comma-separated list of datasets to archive (e.g., "ferc2","ferc6").'
-        default: '"doeiraec","doelead","eia176","eia191","eia757a","eia860","eia860m","eia861","eia923","eia930","eiaaeo","eiamecs","eianems","eiawater","eia_bulk_elec","epacamd_eia","epacems","epapcap","ferc1","ferc2","ferc6","ferc60","ferc714","gridpathratoolkit","mshamines","nrelatb","phmsagas","usgsuspvdb","vcerare"'
+        default: '"doeiraec","doelead","eia176","eia191","eia757a","eia860","eia860m","eia861","eia923","eia930","eiaaeo","eiamecs","eianems","eiawater","eia_bulk_elec","epacamd_eia","epacems","epaegrid","epapcap","ferc1","ferc2","ferc6","ferc60","ferc714","gridpathratoolkit","mshamines","nrelatb","phmsagas","usgsuspvdb","vcerare"'
         required: true
         type: string
       create_github_issue:
@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         # Note that we can't pass global env variables to the matrix, so we manually reproduce the list of datasets here.
-        dataset: ${{ fromJSON(format('[{0}]', inputs.datasets || '"doeiraec","doelead","eia176","eia191","eia757a","eia860","eia860m","eia861","eia923","eia930","eiaaeo","eiamecs","eianems","eiawater","eia_bulk_elec","epacamd_eia","epacems","epapcap","ferc1","ferc2","ferc6","ferc60","ferc714","gridpathratoolkit","mshamines","nrelatb","phmsagas","usgsuspvdb","vcerare"' )) }}
+        dataset: ${{ fromJSON(format('[{0}]', inputs.datasets || '"doeiraec","doelead","eia176","eia191","eia757a","eia860","eia860m","eia861","eia923","eia930","eiaaeo","eiamecs","eianems","eiawater","eia_bulk_elec","epacamd_eia","epacems","epaegrid","epapcap","ferc1","ferc2","ferc6","ferc60","ferc714","gridpathratoolkit","mshamines","nrelatb","phmsagas","usgsuspvdb","vcerare"' )) }}
       fail-fast: false
     runs-on: ubuntu-latest
     permissions:

src/pudl_archiver/archivers/classes.py

@@ -208,6 +208,27 @@ def add_to_archive(self, zip_path: Path, filename: str, blob: typing.BinaryIO):
                 archive=archive, filename=filename, data=blob.read()
             )
 
+    async def download_add_to_archive_and_unlink(
+        self, url: str, filename: str, zip_path: Path
+    ):
+        """Download a file, add it to an zip file in and archive and unlink.
+
+        Little helper function that combines three common steps often repeated together:
+        * :meth:`download_file`
+        * :meth:`add_to_archive`
+        * :meth:`Path.unlink`
+        """
+        download_path = self.download_directory / filename
+        await self.download_file(url, download_path)
+        self.add_to_archive(
+            zip_path=zip_path,
+            filename=filename,
+            blob=download_path.open("rb"),
+        )
+        # Don't want to leave multiple files on disk, so delete
+        # immediately after they're safely stored in the ZIP
+        download_path.unlink()
+
     async def get_json(self, url: str, **kwargs) -> dict[str, str]:
         """Get a JSON and return it as a dictionary."""
         response = await retry_async(self.session.get, args=[url], kwargs=kwargs)

src/pudl_archiver/archivers/epa/epaegrid.py

@@ -0,0 +1,87 @@
+"""Download EPA eGRID data."""
+
+import re
+
+from pudl_archiver.archivers.classes import (
+    AbstractDatasetArchiver,
+    ArchiveAwaitable,
+    ResourceInfo,
+)
+from pudl_archiver.frictionless import ZipLayout
+
+BASE_URL = "https://www.epa.gov/egrid/historical-egrid-data"
+
+
+class EpaEgridArchiver(AbstractDatasetArchiver):
+    """EPA eGrid archiver."""
+
+    name = "epaegrid"
+
+    async def get_resources(self) -> ArchiveAwaitable:
+        """Download EPA eGrid resources."""
+        # All of the "historical" data is stored on one page while the most
+        # recent data is stored on the main dataset page. So we need to
+        # go grab all the old data first and then get the newest data.
+        link_pattern = re.compile(r"egrid(\d{4})_data(_v(\d{1})|).xlsx", re.IGNORECASE)
+        years = []
+        for link in await self.get_hyperlinks(BASE_URL, link_pattern):
+            match = link_pattern.search(link)
+            year = int(match.group(1))
+            years += [year]
+            if self.valid_year(year):
+                yield self.get_year_resource(
+                    year, [BASE_URL, "https://www.epa.gov/egrid/egrid-pm25"]
+                )
+
+        recent_year = max(years) + 1
+        recent_urls = [
+            "https://www.epa.gov/egrid/detailed-data",
+            "https://www.epa.gov/egrid/summary-data",
+            "https://www.epa.gov/egrid/egrid-technical-guide",
+            "https://www.epa.gov/egrid/egrid-pm25",
+        ]
+        if self.valid_year(recent_year):
+            yield self.get_year_resource(recent_year, recent_urls)
+
+    async def get_year_resource(self, year: int, base_urls: list[str]) -> ResourceInfo:
+        """Download all files pertaining to an eGRID year."""
+        zip_path = self.download_directory / f"epaegrid-{year}.zip"
+        table_link_pattern = re.compile(
+            rf"egrid{year}(?:_|-)([a-z,_\d,-]*)(.xlsx|.pdf|.txt)$", re.IGNORECASE
+        )
+        data_paths_in_archive = set()
+        for base_url in base_urls:
+            for url in await self.get_hyperlinks(base_url, table_link_pattern):
+                match = table_link_pattern.search(url)
+                # TODO: this setup leaves in all the _rev# _r# _r#_# and _{date}
+                # in this table name. It would be ideal to remove this all together
+                table = match.group(1).replace("_", "-").lower().strip()
+                file_extension = match.group(2)
+                filename = f"epaegrid-{year}-{table}{file_extension}"
+                await self.download_add_to_archive_and_unlink(url, filename, zip_path)
+                data_paths_in_archive.add(filename)
+        # there is one file with PM 2.5 data in it which says its for 2018-2022
+        # add this file to every one of the yearly zips
+        pm_combo_years = [2018, 2019, 2020, 2021]
+        if year in pm_combo_years:
+            url = "https://www.epa.gov/system/files/documents/2024-06/egrid-draft-pm-emissions.xlsx"
+            filename = f"epaegrid-{year}-pm-emissions.xlsx"
+            await self.download_add_to_archive_and_unlink(url, filename, zip_path)
+            data_paths_in_archive.add(filename)
+        # There are two special case links on the PM 2.5 page that don't adhere to a
+        # clear pattern. so we'll hardcode how to grab them.
+        pm_special_year_links = {
+            2020: "https://www.epa.gov/system/files/documents/2022-12/eGRID2020%20DRAFT%20PM%20Memo.pdf",
+            2019: "https://www.epa.gov/system/files/documents/2023-01/DRAFT%202019%20PM%20Memo.pdf",
+            2018: "https://www.epa.gov/sites/default/files/2020-07/documents/draft_egrid_pm_white_paper_7-20-20.pdf",
+        }
+        if year in pm_special_year_links:
+            url = pm_special_year_links[year]
+            filename = f"epaegrid-{year}-pm-emissions-methodology.pdf"
+            await self.download_add_to_archive_and_unlink(url, filename, zip_path)
+            data_paths_in_archive.add(filename)
+        return ResourceInfo(
+            local_path=zip_path,
+            partitions={"year": year},
+            layout=ZipLayout(file_paths=data_paths_in_archive),
+        )

src/pudl_archiver/cli.py

@@ -34,8 +34,8 @@ def parse_main(args=None):
         nargs="*",
         help="Years to download data for. Supported datasets: censusdp1tract, censuspep, "
         "eia176, eia191, eia757a, eia860, eia860m, eia861, eia923, eia930, eia_bulk_elec, "
-        "eiaaeo, eiamecs, eiawater, epacamd_eia, epacems, ferc1, ferc2, ferc6, ferc60, ferc714, "
-        "mshamines, nrelatb, phmsagas",
+        "eiaaeo, eiamecs, eiawater, epacamd_eia, epacems, epaegrid,ferc1, ferc2, ferc6, "
+        "ferc60, ferc714, mshamines, nrelatb, phmsagas",
         type=int,
     )
     parser.add_argument(

src/pudl_archiver/package_data/zenodo_doi.yaml

@@ -55,6 +55,9 @@ epacamd_eia:
 epacems:
   production_doi: 10.5281/zenodo.10233185
   sandbox_doi: 10.5072/zenodo.12943
+epaegrid:
+  production_doi: 10.5281/zenodo.14767235
+  sandbox_doi: 10.5072/zenodo.159996
 epapcap:
   production_doi: 10.5281/zenodo.14757598
   #sandbox_doi: # Update!!