From ef2033964d56f9ca71cd66c9e7f1ad1b65d0b714 Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Fri, 3 Nov 2023 14:24:02 -0400 Subject: [PATCH 1/2] only fix a reporting_frequency_code when the column exists --- src/pudl/extract/eia923.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pudl/extract/eia923.py b/src/pudl/extract/eia923.py index 87a6c9413f..a55b7cceea 100644 --- a/src/pudl/extract/eia923.py +++ b/src/pudl/extract/eia923.py @@ -50,9 +50,10 @@ def process_raw(self, df, page, **partition): df = remove_leading_zeros_from_numeric_strings(df=df, col_name=col) df = self.add_data_maturity(df, page, **partition) # Fill in blank reporting_frequency_code for monthly data - df.loc[ - df["data_maturity"] == "incremental_ytd", "reporting_frequency_code" - ] = "M" + if "reporting_frequency_code" in df.columns: + df.loc[ + df["data_maturity"] == "incremental_ytd", "reporting_frequency_code" + ] = "M" # the 2021 early release data had some ding dang "."'s and nulls in the year column if "report_year" in df.columns: mask = (df.report_year == ".") | df.report_year.isnull() From acbf7889bd16046ebc7131065a5ccc7abb4301a7 Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Mon, 6 Nov 2023 16:43:34 -0500 Subject: [PATCH 2/2] update tox and eia923 rows --- test/validate/mcoe_test.py | 4 ++-- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/validate/mcoe_test.py b/test/validate/mcoe_test.py index 949b4914ea..1d3ea05d3c 100644 --- a/test/validate/mcoe_test.py +++ b/test/validate/mcoe_test.py @@ -109,8 +109,8 @@ def test_no_null_rows_mcoe(pudl_out_mcoe, live_dbs, df_name, thresh): ("hr_by_unit", 387_694, 32_416), ("hr_by_gen", 599_496, 50_070), ("fuel_cost", 599_496, 50_070), - ("capacity_factor", 5_178_892, 433_277), - ("mcoe", 5_179_300, 433_311), + ("capacity_factor", 5_178_828, 433_286), + ("mcoe", 5_179_236, 433_320), ], ) def test_minmax_rows_mcoe(pudl_out_mcoe, live_dbs, monthly_rows, annual_rows, df_name): diff --git a/tox.ini b/tox.ini index b547c307af..280fab1ddc 100644 --- a/tox.ini +++ b/tox.ini @@ -191,8 +191,8 @@ commands = {[testenv:unit]commands} {[testenv:integration]commands} bash -c 'rm -f tox-nuke.log' - bash -c 'coverage run --append src/pudl/convert/ferc_to_sqlite.py --logfile tox-nuke.log --clobber src/pudl/package_data/settings/etl_full.yml' - bash -c 'coverage run --append src/pudl/cli/etl.py --logfile tox-nuke.log --clobber src/pudl/package_data/settings/etl_full.yml' + bash -c 'coverage run --append src/pudl/ferc_to_sqlite/cli.py --logfile tox-nuke.log --clobber src/pudl/package_data/settings/etl_full.yml' + bash -c 'coverage run --append src/pudl/cli/etl.py --logfile tox-nuke.log src/pudl/package_data/settings/etl_full.yml' pytest {tty:--color=yes} --live-dbs {posargs} {[testenv]covargs} \ --etl-settings src/pudl/package_data/settings/etl_full.yml \ test/integration