EIA 923 energy storage transform (#3546)

* Move the fix_boolean_columns function to the helpers module so that it can be used for EIA923 as well as EIA860. This can probably be combined with the existing convert_col_to_bool function, but keeping both for now * Add beginnings of a _core_eia923_energy_storage table function. Paused to fix the issues with encoding in the EIA modules * Finish transformer for EIA 923 energy storage table: dquote> - Update names of the total columns to match the monthly columns dquote> - Change the table from yearly to monthly dquote> - Add new fields - Add new migration * Update doc strings to fix build fail * Make migrations compatible with main * Small resource/field metadata updates. * minor data cleaning for the EIA-923 energy storage table. * Add foreign_key_rules to the core_eia__codes_storage_technology_types and core_eia__codes_storage_enclosure_types tables -- this should have been part of the EIA860 storage table PR * Clarify description of core_eia923__monthly_energy_storage table * Add migration for new foreign key relationships * Add data_maturity field to the cooling_system table --------- Co-authored-by: Zane Selvans <[email protected]>
catalyst-cooperative · Apr 10, 2024 · d37d630 · d37d630
1 parent 568dbfa
commit d37d630
Show file tree

Hide file tree

Showing 14 changed files with 291 additions and 32 deletions.
diff --git a/docs/release_notes.rst b/docs/release_notes.rst
@@ -26,6 +26,8 @@ New Data Coverage
 * Added new :ref:`core_eia860__scd_generators_energy_storage` table from EIA860 Schedule
   3.4 which contains energy storage generator attributes. See :pr:`3488` and :pr:`3526`.
   which contains solar generator attributes. See :pr:`3524` and :pr:`3482`
+* Added new :ref:`core_eia923__monthly_energy_storage` table from EIA923 which contains
+  monthly energy and fuel consumption metrics. See :pr:`3516` and :pr:`3546`.
 * Integrated the most processed version of the GridPath RA Toolkit wind and solar
   generation profiles, as well as the tables describing how individual generators were
   aggregated together to create the profiles. See issues :issue:`3509,3510,3511,3515`

diff --git a/migrations/versions/172898b30824_add_data_maturity_to_eia923_cooling_.py b/migrations/versions/172898b30824_add_data_maturity_to_eia923_cooling_.py
@@ -0,0 +1,34 @@
+"""Add data_maturity to EIA923 cooling system table
+
+Revision ID: 172898b30824
+Revises: 823e58cbf0e0
+Create Date: 2024-04-10 13:36:48.866852
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '172898b30824'
+down_revision = '823e58cbf0e0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_core_eia923__cooling_system_information', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'))
+        batch_op.create_foreign_key(batch_op.f('fk__core_eia923__cooling_system_information_data_maturity_core_pudl__codes_data_maturities'), 'core_pudl__codes_data_maturities', ['data_maturity'], ['code'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_core_eia923__cooling_system_information', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk__core_eia923__cooling_system_information_data_maturity_core_pudl__codes_data_maturities'), type_='foreignkey')
+        batch_op.drop_column('data_maturity')
+
+    # ### end Alembic commands ###
diff --git a/migrations/versions/4e14f531f5d6_add_eia923_energy_storage_monthly_table.py b/migrations/versions/4e14f531f5d6_add_eia923_energy_storage_monthly_table.py
@@ -0,0 +1,44 @@
+"""Add EIA923 Energy Storage monthly table
+
+Revision ID: 4e14f531f5d6
+Revises: 00e4382815d9
+Create Date: 2024-04-09 11:57:50.890543
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4e14f531f5d6'
+down_revision = '00e4382815d9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('core_eia923__monthly_energy_storage',
+    sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'),
+    sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'),
+    sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'),
+    sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'),
+    sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'),
+    sa.Column('fuel_units', sa.Enum('barrels', 'mcf', 'mwh', 'short_tons'), nullable=True, comment='Reported unit of measure for fuel.'),
+    sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'),
+    sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'),
+    sa.Column('gross_generation_mwh', sa.Float(), nullable=True, comment='Gross electricity generation for the specified period in megawatt-hours (MWh).'),
+    sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'),
+    sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__monthly_energy_storage_data_maturity_core_pudl__codes_data_maturities')),
+    sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia923__monthly_energy_storage_energy_source_code_core_eia__codes_energy_sources')),
+    sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia923__monthly_energy_storage_plant_id_eia_core_eia__entity_plants')),
+    sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_core_eia923__monthly_energy_storage_prime_mover_code_core_eia__codes_prime_movers')),
+    sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_core_eia923__monthly_energy_storage'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('core_eia923__monthly_energy_storage')
+    # ### end Alembic commands ###
diff --git a/migrations/versions/823e58cbf0e0_update_foreign_key_relationships_for_.py b/migrations/versions/823e58cbf0e0_update_foreign_key_relationships_for_.py
@@ -0,0 +1,40 @@
+"""Update foreign key relationships for EIA860 storage coding tables
+
+Revision ID: 823e58cbf0e0
+Revises: 4e14f531f5d6
+Create Date: 2024-04-10 12:50:46.401285
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '823e58cbf0e0'
+down_revision = '4e14f531f5d6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('core_eia860__scd_generators_energy_storage', schema=None) as batch_op:
+        batch_op.create_foreign_key(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_enclosure_code_core_eia__codes_storage_enclosure_types'), 'core_eia__codes_storage_enclosure_types', ['storage_enclosure_code'], ['code'])
+        batch_op.create_foreign_key(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_3_core_eia__codes_storage_technology_types'), 'core_eia__codes_storage_technology_types', ['storage_technology_code_3'], ['code'])
+        batch_op.create_foreign_key(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_4_core_eia__codes_storage_technology_types'), 'core_eia__codes_storage_technology_types', ['storage_technology_code_4'], ['code'])
+        batch_op.create_foreign_key(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_2_core_eia__codes_storage_technology_types'), 'core_eia__codes_storage_technology_types', ['storage_technology_code_2'], ['code'])
+        batch_op.create_foreign_key(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_1_core_eia__codes_storage_technology_types'), 'core_eia__codes_storage_technology_types', ['storage_technology_code_1'], ['code'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('core_eia860__scd_generators_energy_storage', schema=None) as batch_op:
+        batch_op.drop_constraint(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_1_core_eia__codes_storage_technology_types'), type_='foreignkey')
+        batch_op.drop_constraint(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_2_core_eia__codes_storage_technology_types'), type_='foreignkey')
+        batch_op.drop_constraint(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_4_core_eia__codes_storage_technology_types'), type_='foreignkey')
+        batch_op.drop_constraint(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_technology_code_3_core_eia__codes_storage_technology_types'), type_='foreignkey')
+        batch_op.drop_constraint(batch_op.f('fk_core_eia860__scd_generators_energy_storage_storage_enclosure_code_core_eia__codes_storage_enclosure_types'), type_='foreignkey')
+
+    # ### end Alembic commands ###
diff --git a/src/pudl/helpers.py b/src/pudl/helpers.py
@@ -1779,6 +1779,25 @@ def convert_col_to_bool(
     return df
 
 
+def fix_boolean_columns(
+    df: pd.DataFrame,
+    boolean_columns_to_fix: list[str],
+) -> pd.DataFrame:
+    """Fix standard issues with EIA boolean columns.
+
+    Most boolean columns have either "Y" for True or "N" for False. A subset of the
+    columns have "X" values which represents a False value. A subset of the columns
+    have "U" values, presumably for "Unknown," which must be set to null in order to
+    convert the columns to datatype Boolean.
+    """
+    fillna_cols = {col: pd.NA for col in boolean_columns_to_fix}
+    boolean_replace_cols = {
+        col: {"Y": True, "N": False, "X": False, "U": pd.NA}
+        for col in boolean_columns_to_fix
+    }
+    return df.fillna(fillna_cols).replace(to_replace=boolean_replace_cols)
+
+
 def scale_by_ownership(
     gens: pd.DataFrame,
     own_eia860: pd.DataFrame,

diff --git a/src/pudl/metadata/fields.py b/src/pudl/metadata/fields.py
@@ -1139,6 +1139,7 @@
             "A 2-3 letter code indicating the energy source (e.g. fuel type) "
             "associated with the record."
         ),
+        # Should this have an enum reference to the core_eia__codes_energy_sources table??
     },
     "energy_source_code_num": {
         "type": "string",
@@ -1737,6 +1738,11 @@
         "description": "Plant's grid voltage at point of interconnection to transmission or distibution facilities",
         "unit": "kV",
     },
+    "gross_generation_mwh": {
+        "type": "number",
+        "description": "Gross electricity generation for the specified period in megawatt-hours (MWh).",
+        "unit": "MWh",
+    },
     "gross_load_mw": {
         "type": "number",
         "description": "Average power in megawatts delivered during time interval measured.",

diff --git a/src/pudl/metadata/labels.py b/src/pudl/metadata/labels.py
@@ -70,6 +70,4 @@
 The physical units fuel consumption is reported in.  All consumption is reported in
 either short tons for solids, thousands of cubic feet for gases, and barrels for
 liquids.
-
-Not currently being used.
 """
diff --git a/src/pudl/metadata/resources/eia.py b/src/pudl/metadata/resources/eia.py
@@ -470,6 +470,9 @@
                     ["energy_source_code_4"],
                     ["energy_source_code_5"],
                     ["energy_source_code_6"],
+                    ["energy_source_code_7"],
+                    ["energy_source_code_8"],
+                    ["energy_source_code_9"],
                     ["startup_source_code_1"],
                     ["startup_source_code_2"],
                     ["startup_source_code_3"],
@@ -1164,6 +1167,7 @@
         "schema": {
             "fields": ["code", "label", "description"],
             "primary_key": ["code"],
+            "foreign_key_rules": {"fields": [["storage_enclosure_code"]]},
         },
         "encoder": CODE_METADATA["core_eia__codes_storage_enclosure_types"],
         "field_namespace": "eia",
@@ -1175,6 +1179,14 @@
         "schema": {
             "fields": ["code", "label", "description"],
             "primary_key": ["code"],
+            "foreign_key_rules": {
+                "fields": [
+                    ["storage_technology_code_1"],
+                    ["storage_technology_code_2"],
+                    ["storage_technology_code_3"],
+                    ["storage_technology_code_4"],
+                ]
+            },
         },
         "encoder": CODE_METADATA["core_eia__codes_storage_technology_types"],
         "field_namespace": "eia",

diff --git a/src/pudl/metadata/resources/eia860.py b/src/pudl/metadata/resources/eia860.py
@@ -112,6 +112,7 @@
                     "core_eia923__monthly_boiler_fuel",
                     "out_eia923__boiler_fuel",
                     "out_eia923__monthly_boiler_fuel",
+                    "core_eia923__monthly_energy_storage",
                 ],
             },
         },
@@ -242,6 +243,7 @@
                     "_out_eia__monthly_derived_generator_attributes",
                     "out_eia__monthly_generators",
                     "core_eia860m__changelog_generators",
+                    "core_eia923__monthly_energy_storage",
                 ],
             },
         },
@@ -396,6 +398,7 @@
                     "_out_eia__monthly_derived_generator_attributes",
                     "out_eia__monthly_generators",
                     "core_eia860m__changelog_generators",
+                    "core_eia923__monthly_energy_storage",
                 ],
             },
         },

diff --git a/src/pudl/metadata/resources/eia923.py b/src/pudl/metadata/resources/eia923.py
@@ -123,6 +123,14 @@
 generation unit level up to the plant prime mover level, so as to be compatible with
 fossil fuel generation data."""
     ),
+    "core_eia923__monthly_energy_storage": (
+        """EIA-923 Monthly Generation and Fuel Consumption Time Series. From EIA-923 Schedule 3.
+
+Monthly quantities of energy consumed and discharged ("generated") by energy storage
+units. The total MWh discharged from the energy storage unit during the
+reporting period is the gross generation and the difference between gross generation
+and consumption is the net generation."""
+    ),
 }
 
 RESOURCE_METADATA: dict[str, dict[str, Any]] = {
@@ -710,6 +718,7 @@
                 "monthly_total_withdrawal_volume_gallons",
                 "annual_total_chlorine_lbs",
                 "monthly_total_chlorine_lbs",
+                "data_maturity",
             ],
             "primary_key": [
                 "plant_id_eia",
@@ -754,6 +763,32 @@
         "sources": ["eia923"],
         "etl_group": "eia923",
     },
+    "core_eia923__monthly_energy_storage": {
+        "description": TABLE_DESCRIPTIONS["core_eia923__monthly_energy_storage"],
+        "schema": {
+            "fields": [
+                "plant_id_eia",
+                "report_date",
+                "prime_mover_code",
+                "energy_source_code",
+                "data_maturity",
+                "fuel_units",
+                "fuel_consumed_for_electricity_units",
+                "fuel_consumed_units",
+                "gross_generation_mwh",
+                "net_generation_mwh",
+            ],
+            "primary_key": [
+                "plant_id_eia",
+                "report_date",
+                "prime_mover_code",
+                "energy_source_code",
+            ],
+        },
+        "field_namespace": "eia",
+        "sources": ["eia923"],
+        "etl_group": "eia923",
+    },
 }
 """EIA-923 resource attributes organized by PUDL identifier (``resource.name``).
 

diff --git a/src/pudl/package_data/eia923/column_maps/energy_storage.csv b/src/pudl/package_data/eia923/column_maps/energy_storage.csv
@@ -15,7 +15,7 @@ energy_source_code,reported_fuel_type_code,reported_fuel_type_code,reported_fuel
 fuel_type_code_aer,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code,aer_fuel_type_code
 balancing_authority_code_eia,,,,,balancing_authority_code,balancing_authority_code,balancing_authority_code,balancing_authority_code,balancing_authority_code,balancing_authority_code
 reporting_frequency_code,,,,,respondent_frequency,respondent_frequency,respondent_frequency,respondent_frequency,respondent_frequency,
-fuel_unit,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label
+fuel_units,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label,physical_unit_label
 fuel_consumed_units_january,quantity_january,quantity_january,quantity_january,quantity_january,quantity_january,quantity_january,quantity_january,quantity_january,quantity_january,quantity_january
 fuel_consumed_units_february,quantity_february,quantity_february,quantity_february,quantity_february,quantity_february,quantity_february,quantity_february,quantity_february,quantity_february,quantity_february
 fuel_consumed_units_march,quantity_march,quantity_march,quantity_march,quantity_march,quantity_march,quantity_march,quantity_march,quantity_march,quantity_march,quantity_march
@@ -64,8 +64,8 @@ net_generation_mwh_september,netgen_september,netgen_september,netgen_september,
 net_generation_mwh_october,netgen_october,netgen_october,netgen_october,netgen_october,netgen_october,netgen_october,netgen_october,netgen_october,netgen_october,netgen_october
 net_generation_mwh_november,netgen_november,netgen_november,netgen_november,netgen_november,netgen_november,netgen_november,netgen_november,netgen_november,netgen_november,netgen_november
 net_generation_mwh_december,netgen_december,netgen_december,netgen_december,netgen_december,netgen_december,netgen_december,netgen_december,netgen_december,netgen_december,netgen_december
-total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity
-electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity
-gross_generation_mwh,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours
+total_fuel_consumed_units,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity,total_fuel_consumption_quantity
+total_fuel_consumed_for_electricity_units,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity,electric_fuel_consumption_quantity
+total_gross_generation_mwh,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours,gross_generation_megawatthours
 total_net_generation_mwh,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours,net_generation_megawatthours
 report_year,year,year,year,year,year,year,year,year,year,year