diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 848cb70482..e87113f171 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -44,6 +44,8 @@ EIA 860 EIA 923 ~~~~~~~ * Added EIA 923 early release data from 2023. See :issue:`3719` and PR :pr:`3721`. +* Added EIA 923 monthly data through May as part of the Q2 quarterly release. See + :issue:`3760` and :pr:`3768`. EPA CEMS ~~~~~~~~ diff --git a/docs/templates/eia923_child.rst.jinja b/docs/templates/eia923_child.rst.jinja index 3588de9c2c..c82855acfd 100644 --- a/docs/templates/eia923_child.rst.jinja +++ b/docs/templates/eia923_child.rst.jinja @@ -91,4 +91,38 @@ Data Estimates Plants that did not respond or reported unverified data were recorded as estimates rolled in with the state/fuel aggregates values reported under the plant id 99999. +Boiler Fuel Primary Keys +------------------------ +The :ref:`core_eia923__monthly_boiler_fuel` table has several sneaky primary keys and duplicate rows. +The main primary keys for the table are: ``plant_id_eia, boiler_id, energy_source_code, prime_mover_code, +report_date``. There are some rows that also differ based on ``associated_combined_heat_power``, due +to mid-year retirement of units that are assocated with combine heat and power systems, and +``operator_name``, due to lenient standards for string columns (the all have the same ``operator_id`` +value). We drop both the ``associated_combined_heat_power`` and ``operator_name`` fields from the final +normalized table, causing duplicate rows. Luckily, these rows don't provide any conflicting information. +Because they are the same plant, when one row contains an NA value, the other contains a numeric value. +We can easily drop duplicates based on which rows contain NA values with no duplicate value reconciling +necessary. + +There are still more duplicate rows with identical qualitative plant information. Luckily, none of these +duplicates contain conflicting information either. All duplicate rows have at least one row containing +solely NA and 0 values. + +To address both issues at once, we drop all the duplicate rows with NA or 0 values in the non primary +key columns. One side affect of this is that duplicate rows where both rows contain NA and 0 values will +both get dropped. This leads to gaps in the data where certain months are missing. These values can be +assumed to be 0 or NA. + +Boiler Fuel Years +----------------- +The :ref:`core_eia923__monthly_boiler_fuel` table reports all months in a given year, even if there is +no data. At present, we haven't truncated the data after the most recently integrated month, so you will +see all months. + +Fluctuations in row count between each quarterly update are therefore due to changes in primary key +quirks as described above. + + + + {%- endblock %} diff --git a/src/pudl/package_data/eia923/file_map.csv b/src/pudl/package_data/eia923/file_map.csv index 61b5218b5f..e61abb2fcc 100644 --- a/src/pudl/package_data/eia923/file_map.csv +++ b/src/pudl/package_data/eia923/file_map.csv @@ -1,16 +1,16 @@ page,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024 -boiler_fuel,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx +boiler_fuel,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx coal_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,-1,-1,-1,-1 -energy_storage,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -fuel_receipts_costs,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -generation_fuel,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -generator,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -oil_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx +energy_storage,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +fuel_receipts_costs,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +generation_fuel,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +generator,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +oil_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx petcoke_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,-1,-1,-1,-1 -plant_frame,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -puerto_rico,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -stocks,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx -plant_frame_puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_04_2024_21JUN2024.xlsx +plant_frame,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +puerto_rico,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +stocks,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx +plant_frame_puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Early_Release.xlsx,EIA923_Schedules_2_3_4_5_M_05_2024_18JUL2024.xlsx emissions_control,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedule_8_Annual_Environmental_Information_2012_Final_Revision.xlsx,EIA923_Schedule_8_PartsA-D_EnvData_2013_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2014_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2015_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2016_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Envir_Infor_2017_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2018_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2019_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2020_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2021_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2022_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2023_Early_Release.xlsx,-1 byproduct_expenses_and_revenues,-1,-1,-1,-1,-1,-1,-1,SCHEDULE 3A 5A 8A 8B 8C 8D 8E 8F 2008.xlsm,SCHEDULE 3A 5A 8A 8B 8C 8D 8E 8F REVISED 2009 04112011.xls,SCHEDULE 3A 5A 8A 8B 8C 8D 8E 8F 2010 on NOV 30 2011.xls,EIA923_Schedule_8_PartsA-F_EnvData_2011_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2012_Final_Revision.xlsx,EIA923_Schedule_8_PartsA-D_EnvData_2013_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2014_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2015_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2016_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Envir_Infor_2017_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2018_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2019_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2020_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2021_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2022_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2023_Early_Release.xlsx,-1 byproduct_disposition,-1,-1,-1,-1,-1,-1,-1,SCHEDULE 3A 5A 8A 8B 8C 8D 8E 8F 2008.xlsm,SCHEDULE 3A 5A 8A 8B 8C 8D 8E 8F REVISED 2009 04112011.xls,SCHEDULE 3A 5A 8A 8B 8C 8D 8E 8F 2010 on NOV 30 2011.xls,EIA923_Schedule_8_PartsA-F_EnvData_2011_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2012_Final_Revision.xlsx,EIA923_Schedule_8_PartsA-D_EnvData_2013_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2014_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2015_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2016_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Envir_Infor_2017_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2018_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2019_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2020_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2021_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2022_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2023_Early_Release.xlsx,-1 diff --git a/src/pudl/package_data/glue/pudl_id_mapping.xlsx b/src/pudl/package_data/glue/pudl_id_mapping.xlsx index b41bcf533c..3f24dcbcb1 100644 Binary files a/src/pudl/package_data/glue/pudl_id_mapping.xlsx and b/src/pudl/package_data/glue/pudl_id_mapping.xlsx differ diff --git a/src/pudl/package_data/glue/utility_id_pudl.csv b/src/pudl/package_data/glue/utility_id_pudl.csv index d3c42908e2..2ec8391f9d 100644 --- a/src/pudl/package_data/glue/utility_id_pudl.csv +++ b/src/pudl/package_data/glue/utility_id_pudl.csv @@ -16342,3 +16342,36 @@ utility_id_pudl,utility_id_ferc1,utility_name_ferc1,utility_id_eia,utility_name_ 16384,,,66290,NSF Energy One LLC 16385,,,66291,Portage Solar Plant 16386,,,66292,Desert Willow Energy Storage +16387,,,66317,"Kola Energy Storage, LLC" +16388,,,66336,"Wild Plains Wind Project, LLC" +16389,,,66352,SMT Ironman BESS LLC +16390,,,66345,"Sebree Solar, LLC" +16391,,,66354,"Anole Energy Storage, LLC" +16392,,,66351,Citadel BESS LLC +16393,,,66348,"Silver State South Storage, LLC" +16394,,,65860,"Madison Fields Solar Project, LLC" +16395,,,66346,"Silver Peak Solar, LLC" +16396,,,66314,JGT2 Energy LLC +16397,,,66318,"Zeta Solar, LLC" +16398,,,66334,Twin Lakes Solar LLC +16399,,,66319,"Heartwood Solar, LLC" +16400,,,66360,Reliability Design & Development LLC +16401,,,66320,"White Tail Solar, LLC" +16402,,,66350,Wigeon Whistle BESS LLC +16403,,,66338,Al Pastor BESS LLC +16404,,,66316,"Northumberland Solar I, LLC" +16405,,,66331,"Birch Creek Power, LLC" +16406,,,66347,"Placid Solar II, LLC" +16407,,,66335,REV Renewables LLC +16408,,,66294,"NSF Torrey Site 2, LLC" +16409,,,66300,NY CDG Genesee 1 LLC +16410,,,66321,NY CDG Montgomery 1 LLC +16411,,,66293,"NSF Torrey Site 3, LLC" +16412,,,66295,"NSF Torrey Site 1, LLC" +16413,,,66301,NY CDG Genesee 4 LLC +16414,,,66342,"Catalyze Joliet 1101 Cherry Hill Road Microgrid, LLC" +16415,,,66305,"Rio Vista Executive Boat & RV Storage, LLC" +16416,,,66304,PFMD LL Baltimore LLC +16417,,,66303,PFMD LL Jessup LLC +16418,,,66306,Town Of Cary +16419,,,66343,"Catalyze Rochelle Wiscold Drive Microgrid, LLC" diff --git a/src/pudl/transform/eia923.py b/src/pudl/transform/eia923.py index fcf9a966c5..64cf1b33e2 100644 --- a/src/pudl/transform/eia923.py +++ b/src/pudl/transform/eia923.py @@ -833,6 +833,11 @@ def _core_eia923__boiler_fuel(raw_eia923__boiler_fuel: pd.DataFrame) -> pd.DataF * Create a fuel_type_code_pudl field that organizes fuel types into clean, distinguishable categories. * Combine year and month columns into a single date column. + * Drop duplicate rows with NA or 0 in all value columns. + + Eventually we should truncate this table by the last year-month that was integrated. + Right now all months get integrated for a given year, regardless of whether there's + data for them. Args: raw_eia923__boiler_fuel: The raw ``raw_eia923__boiler_fuel`` dataframe. diff --git a/src/pudl/workspace/datastore.py b/src/pudl/workspace/datastore.py index 09ed45b44d..24de482b51 100644 --- a/src/pudl/workspace/datastore.py +++ b/src/pudl/workspace/datastore.py @@ -193,7 +193,7 @@ class ZenodoDoiSettings(BaseSettings): eia860: ZenodoDoi = "10.5281/zenodo.11662381" eia860m: ZenodoDoi = "10.5281/zenodo.11110602" eia861: ZenodoDoi = "10.5281/zenodo.10204708" - eia923: ZenodoDoi = "10.5281/zenodo.12656894" + eia923: ZenodoDoi = "10.5281/zenodo.12721286" eia930: ZenodoDoi = "10.5281/zenodo.10840078" eiawater: ZenodoDoi = "10.5281/zenodo.10806016" eiaaeo: ZenodoDoi = "10.5281/zenodo.10838488" diff --git a/test/validate/eia_test.py b/test/validate/eia_test.py index 1977960641..e7bec0563a 100644 --- a/test/validate/eia_test.py +++ b/test/validate/eia_test.py @@ -46,17 +46,17 @@ def test_no_null_cols_eia(pudl_out_eia, live_dbs, cols, df_name): @pytest.mark.parametrize( "df_name,raw_rows,monthly_rows,annual_rows", [ - ("bf_eia923", 1_642_829, 1_642_829, 135_980), + ("bf_eia923", 1_642_806, 1_642_806, 135_980), ("bga_eia860", 153_487, 153_487, 153_487), - ("boil_eia860", 89_051, 89_051, 89_051), + ("boil_eia860", 89_050, 89_050, 89_050), ("frc_eia923", 673_343, 274_479, 26_709), ("gen_eia923", None, 5_494_932, 459_711), - ("gens_eia860", 590_881, 590_881, 590_881), - ("gf_eia923", 3_064_042, 3_064_042, 260_842), + ("gens_eia860", 591_256, 591_256, 591_256), + ("gf_eia923", 3_064_045, 3_064_045, 260_842), ("own_eia860", 95_104, 95_104, 95_104), - ("plants_eia860", 215_884, 215_884, 215_884), - ("pu_eia860", 214_965, 214_965, 214_965), - ("utils_eia860", 147_877, 147_877, 147_877), + ("plants_eia860", 216_206, 216_206, 216_206), + ("pu_eia860", 215_288, 215_288, 215_288), + ("utils_eia860", 147_922, 147_922, 147_922), ("emissions_control_equipment_eia860", 62_102, 62_102, 62_102), ("denorm_emissions_control_equipment_eia860", 62_102, 62_102, 62_102), ("boiler_emissions_control_equipment_assn_eia860", 83_977, 83_977, 83_977),