Integrate 2024 Half 2 data for EIA 930 (#3789)

* Update doi, working_partitions, and column maps to include 2024 half 2 * Update release notes
catalyst-cooperative · Aug 13, 2024 · 0a231d5 · 0a231d5
1 parent ec8f4ff
commit 0a231d5
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 64 deletions.
diff --git a/docs/release_notes.rst b/docs/release_notes.rst
@@ -47,6 +47,11 @@ EIA 923
 * Added EIA 923 monthly data through May as part of the Q2 quarterly release. See
   :issue:`3760` and :pr:`3768`.
 
+EIA 930
+~~~~~~~
+* Added EIA 930 hourly data through the end of July as part of the Q2 quarterly release.
+  See :issue:`3761`` and :pr:`3789`.
+
 EPA CEMS
 ~~~~~~~~
 * Added 2024 Q2 of CEMS data. See :issue:`3762` and :pr:`3769`.

diff --git a/src/pudl/metadata/sources.py b/src/pudl/metadata/sources.py
@@ -303,7 +303,7 @@
         "working_partitions": {
             "half_years": [
                 f"{year}half{half}" for year in range(2015, 2025) for half in [1, 2]
-            ][1:-1]  # Begins in H2 of 2015 and currently ends in H1 of 2024
+            ][1:]  # Begins in H2 of 2015 and currently ends in H2 of 2024
         },
         "contributors": [
             CONTRIBUTORS["catalyst-cooperative"],

diff --git a/src/pudl/package_data/eia930/column_maps/balance.csv b/src/pudl/package_data/eia930/column_maps/balance.csv
diff --git a/src/pudl/package_data/eia930/column_maps/interchange.csv b/src/pudl/package_data/eia930/column_maps/interchange.csv
@@ -1,10 +1,10 @@
-half_year_index,2015half2,2016half1,2016half2,2017half1,2017half2,2018half1,2018half2,2019half1,2019half2,2020half1,2020half2,2021half1,2021half2,2022half1,2022half2,2023half1,2023half2,2024half1
-balancing_authority_code_eia,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority
-report_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date
-report_hour_local,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number
-balancing_authority_code_adjacent_eia,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority
-interchange_reported_mwh,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw
-datetime_local,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour
-datetime_utc,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour
-balancing_authority_region_code_eia,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region
-region_code_adjacent_eia930,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region
+half_year_index,2015half2,2016half1,2016half2,2017half1,2017half2,2018half1,2018half2,2019half1,2019half2,2020half1,2020half2,2021half1,2021half2,2022half1,2022half2,2023half1,2023half2,2024half1,2024half2
+balancing_authority_code_eia,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority
+report_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date
+report_hour_local,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number
+balancing_authority_code_adjacent_eia,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority,directly_interconnected_balancing_authority
+interchange_reported_mwh,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw,interchange_mw
+datetime_local,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour
+datetime_utc,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour
+balancing_authority_region_code_eia,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region,region
+region_code_adjacent_eia930,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region,diba_region
diff --git a/src/pudl/package_data/eia930/column_maps/subregion.csv b/src/pudl/package_data/eia930/column_maps/subregion.csv
@@ -1,8 +1,8 @@
-half_year_index,2018half2,2019half1,2019half2,2020half1,2020half2,2021half1,2021half2,2022half1,2022half2,2023half1,2023half2,2024half1
-balancing_authority_code_eia,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority
-report_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date
-report_hour_local,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number
-balancing_authority_subregion_code_eia,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region
-demand_reported_mwh,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw
-datetime_local,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour
-datetime_utc,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour
+half_year_index,2018half2,2019half1,2019half2,2020half1,2020half2,2021half1,2021half2,2022half1,2022half2,2023half1,2023half2,2024half1,2024half2
+balancing_authority_code_eia,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority,balancing_authority
+report_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date,data_date
+report_hour_local,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number,hour_number
+balancing_authority_subregion_code_eia,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region,sub_region
+demand_reported_mwh,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw,demand_mw
+datetime_local,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour,local_time_at_end_of_hour
+datetime_utc,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour,utc_time_at_end_of_hour
diff --git a/src/pudl/package_data/settings/etl_fast.yml b/src/pudl/package_data/settings/etl_fast.yml
@@ -81,7 +81,7 @@ datasets:
           2022,
         ]
     eia930:
-      half_years: ["2024half1"]
+      half_years: ["2024half2"]
     eiaaeo:
       years: [2023] # we can only handle 1 year at a time right now.
   epacems:

diff --git a/src/pudl/workspace/datastore.py b/src/pudl/workspace/datastore.py
@@ -194,7 +194,7 @@ class ZenodoDoiSettings(BaseSettings):
     eia860m: ZenodoDoi = "10.5281/zenodo.11110602"
     eia861: ZenodoDoi = "10.5281/zenodo.10204708"
     eia923: ZenodoDoi = "10.5281/zenodo.12721286"
-    eia930: ZenodoDoi = "10.5281/zenodo.10840078"
+    eia930: ZenodoDoi = "10.5281/zenodo.13149087"
     eiawater: ZenodoDoi = "10.5281/zenodo.10806016"
     eiaaeo: ZenodoDoi = "10.5281/zenodo.10838488"
     eia_bulk_elec: ZenodoDoi = "10.5281/zenodo.13149083"