From e85ce00c5ed655132718cc048db2892e57f890ed Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Wed, 20 Sep 2023 14:37:17 -0600 Subject: [PATCH 01/20] preliminary version of standardization of the calc metrics --- src/pudl/output/ferc1.py | 74 ++++---------------- src/pudl/transform/ferc1.py | 135 ++++++++++++++++++++---------------- 2 files changed, 92 insertions(+), 117 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 9ff916d221..8d5ad025db 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1469,64 +1469,20 @@ def generate_intertable_calculations( f"{self.root_table}: Reconcile inter-table calculations: " f"{list(calculations_intertable.xbrl_factoid.unique())}." ) - # compile the lists of columns we are going to use later - calc_component_idx = ["table_name", "xbrl_factoid"] + self.other_dimensions - # Merge the reported data and the calculation component metadata to enable - # validation of calculated values. Here the data table exploded is supplying the - # values associated with individual calculation components, and the table_name - # and xbrl_factoid to which we aggregate are coming from the calculation - # components table. After merging we use the weights to adjust the reported - # values so they can be summed directly. This gives us aggregated calculated - # values that can later be compared to the higher level reported values. - - # the validation is one_many in all instances expect for the xbrl_factoid - # construction_work_in_progress in the balance_sheet_assets_ferc1 explosion. - # this may be a problem in the calculations that we should track down in #2717 - validate = ( - "one_to_many" - if self.root_table != "balance_sheet_assets_ferc1" - else "many_to_many" - ) - # we are going to merge the data onto the calc components with the _parent - # column names, so the groupby after the merge needs a set of by cols with the - # _parent suffix - meta_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] - gby_parent = [ - f"{col}_parent" if col in meta_idx else col for col in self.exploded_pks - ] - calc_df = ( - pd.merge( - calculations_intertable, - exploded, - validate=validate, - on=calc_component_idx, - ) - # apply the weight from the calc to convey the sign before summing. - .assign(calculated_amount=lambda x: x[self.value_col] * x.weight) - .groupby(gby_parent, as_index=False, dropna=False)[["calculated_amount"]] - .sum(min_count=1) - ) - # remove the _parent suffix so we can merge these calculated values back onto - # the data using the original pks - calc_df.columns = calc_df.columns.str.removesuffix("_parent") - calculated_df = pd.merge( - exploded, - calc_df, - on=self.exploded_pks, - how="outer", - validate="1:1", - indicator=True, - ) - - assert calculated_df[ - (calculated_df._merge == "right_only") - & (calculated_df[self.value_col].notnull()) - ].empty - - calculated_df = calculated_df.drop(columns=["_merge"]) - # Force value_col to be a float to prevent any hijinks with calculating differences. - calculated_df[self.value_col] = calculated_df[self.value_col].astype(float) - + calc_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] + calculated_df = pudl.transform.ferc1.calculate_values_from_components( + calculation_components=calculations_intertable, + data=exploded, + validate="one_to_many", + calc_idx=calc_idx, + value_col=self.value_col, + ) + calculated_df = pudl.transform.ferc1.check_calculcation_metrics( + calculated_df=calculated_df, + value_col=self.value_col, + calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, + table_name=self.root_table, + ) return calculated_df def reconcile_intertable_calculations( @@ -1588,7 +1544,7 @@ def reconcile_intertable_calculations( f"of {calculation_tolerance:.1%}." ) - # # We'll only get here if the proportion of calculations that are off is acceptable + # We'll only get here if the proportion of calculations that are off is acceptable if off_ratio > 0 or np.isnan(off_ratio): logger.info( f"{self.root_table}: has {len(off_df)} ({off_ratio:.02%}) records whose " diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 8b29c4bcbe..9d90533e13 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -24,23 +24,16 @@ from pydantic import validator import pudl -from pudl.analysis.classify_plants_ferc1 import ( - plants_steam_assign_plant_ids, - plants_steam_validate_ids, -) +from pudl.analysis.classify_plants_ferc1 import (plants_steam_assign_plant_ids, + plants_steam_validate_ids) from pudl.extract.ferc1 import TABLE_NAME_MAP_FERC1 from pudl.helpers import convert_cols_dtypes from pudl.metadata.fields import apply_pudl_dtypes from pudl.settings import Ferc1Settings -from pudl.transform.classes import ( - AbstractTableTransformer, - InvalidRows, - RenameColumns, - TableTransformParams, - TransformParams, - cache_df, - enforce_snake_case, -) +from pudl.transform.classes import (AbstractTableTransformer, InvalidRows, + RenameColumns, TableTransformParams, + TransformParams, cache_df, + enforce_snake_case) logger = pudl.logging_helpers.get_logger(__name__) @@ -843,52 +836,13 @@ def reconcile_table_calculations( validate="one_to_many", calc_idx=calc_idx, value_col=params.column_to_check, - ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) - - calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[params.column_to_check] - x.calculated_amount), - rel_diff=lambda x: np.where( - (x[params.column_to_check] != 0.0), - abs(x.abs_diff / x[params.column_to_check]), - np.nan, - ), ) - - off_df = calculated_df[ - ~np.isclose( - calculated_df.calculated_amount, calculated_df[params.column_to_check] - ) - & (calculated_df["abs_diff"].notnull()) - ] - calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] - off_ratio = len(off_df) / len(calculated_values) - - if off_ratio > params.calculation_tolerance: - raise AssertionError( - f"Calculations in {table_name} are off by {off_ratio}. Expected tolerance " - f"of {params.calculation_tolerance}." - ) - - # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0: - logger.info( - f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " - "calculations don't match. Adding correction records to make calculations " - "match reported values." - ) - corrections = off_df.copy() - corrections[params.column_to_check] = ( - corrections[params.column_to_check].fillna(0.0) - - corrections["calculated_amount"] - ) - corrections[xbrl_factoid_name] = corrections[xbrl_factoid_name] + "_correction" - corrections["row_type_xbrl"] = "correction" - corrections["is_within_table_calc"] = True - corrections["record_id"] = pd.NA - - calculated_df = pd.concat( - [calculated_df, corrections], axis="index" - ).reset_index() + calculated_df = check_calculcation_metrics( + calculated_df=calculated_df, + value_col=params.column_to_check, + calculation_tolerance=params.calculation_tolerance, + table_name=table_name, + ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) # Check that sub-total calculations sum to total. if params.subtotal_column is not None: @@ -1001,6 +955,71 @@ def calculate_values_from_components( return calculated_df +def check_calculcation_metrics( + calculated_df: pd.DataFrame, + value_col: str, + calculation_tolerance: float, + table_name: str, +) -> pd.DataFrame: + """Run the calculation metrics and determine if calculations are within tolerance.""" + # Data types were very messy here, including pandas Float64 for the + # calculated_amount columns which did not work with the np.isclose(). Not sure + # why these are cropping up. + calculated_df = calculated_df.convert_dtypes(convert_floating=False).astype( + {value_col: "float64", "calculated_amount": "float64"} + ) + calculated_df = calculated_df.assign( + abs_diff=lambda x: abs(x[value_col] - x.calculated_amount), + rel_diff=lambda x: np.where( + (x[value_col] != 0.0), + abs(x.abs_diff / x[value_col]), + np.nan, + ), + ) + + off_df = calculated_df[ + ~np.isclose(calculated_df.calculated_amount, calculated_df[value_col]) + & (calculated_df["abs_diff"].notnull()) + ] + calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] + if calculated_values.empty: + # Will only occur if all reported values are NaN when calculated values + # exist, or vice versa. + logger.warning( + "Warning: No calculated values have a corresponding reported value in the table." + ) + off_ratio = np.nan + else: + off_ratio = len(off_df) / len(calculated_values) + if off_ratio > calculation_tolerance: + raise AssertionError( + f"Calculations in {table_name} are off by {off_ratio:.2%}. Expected tolerance " + f"of {calculation_tolerance:.1%}." + ) + + # We'll only get here if the proportion of calculations that are off is acceptable + if off_ratio > 0 or np.isnan(off_ratio): + logger.info( + f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " + "calculations don't match. Adding correction records to make calculations " + "match reported values." + ) + corrections = off_df.copy() + corrections[value_col] = ( + corrections[value_col].fillna(0.0) - corrections["calculated_amount"] + ) + corrections["original_factoid"] = corrections["xbrl_factoid"] + corrections["xbrl_factoid"] = corrections["xbrl_factoid"] + "_correction" + corrections["row_type_xbrl"] = "correction" + corrections["is_within_table_calc"] = False + corrections["record_id"] = pd.NA + + calculated_df = pd.concat( + [calculated_df, corrections], axis="index" + ).reset_index() + return calculated_df + + class Ferc1TableTransformParams(TableTransformParams): """A model defining what TransformParams are allowed for FERC Form 1. From 8a242a39ca316d9d62381fcf3ec2b114fff6a9b8 Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Wed, 20 Sep 2023 14:39:13 -0600 Subject: [PATCH 02/20] remove old thangs --- src/pudl/output/ferc1.py | 87 ---------------------------------------- 1 file changed, 87 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 8d5ad025db..b083b04ce9 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1369,10 +1369,6 @@ def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame exploded = ( self.initial_explosion_concatenation(tables_to_explode) .pipe(self.generate_intertable_calculations) - .pipe( - self.reconcile_intertable_calculations, - self.calculation_tolerance.intertable_calculation_errors, - ) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... @@ -1485,89 +1481,6 @@ def generate_intertable_calculations( ) return calculated_df - def reconcile_intertable_calculations( - self: Self, calculated_df: pd.DataFrame, calculation_tolerance: float = 0.05 - ): - """Ensure inter-table calculated values match reported values within a tolerance. - - In addition to checking whether all reported "calculated" values match the output - of our repaired calculations, this function adds a correction record to the - dataframe that is included in the calculations so that after the fact the - calculations match exactly. This is only done when the fraction of records that - don't match within the tolerances of :func:`numpy.isclose` is below a set - threshold. - - Note that only calculations which are off by a significant amount result in the - creation of a correction record. Many calculations are off from the reported values - by exaclty one dollar, presumably due to rounding errrors. These records typically - do not fail the :func:`numpy.isclose()` test and so are not corrected. - - Args: - calculated_df: table with calculated fields - calculation_tolerance: What proportion (0-1) of calculated values are - allowed to be incorrect without raising an AssertionError. - """ - if "calculated_amount" not in calculated_df.columns: - return calculated_df - - # Data types were very messy here, including pandas Float64 for the - # calculated_amount columns which did not work with the np.isclose(). Not sure - # why these are cropping up. - calculated_df = calculated_df.convert_dtypes(convert_floating=False).astype( - {self.value_col: "float64", "calculated_amount": "float64"} - ) - calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[self.value_col] - x.calculated_amount), - rel_diff=lambda x: np.where( - (x[self.value_col] != 0.0), - abs(x.abs_diff / x[self.value_col]), - np.nan, - ), - ) - off_df = calculated_df[ - ~np.isclose(calculated_df.calculated_amount, calculated_df[self.value_col]) - & (calculated_df["abs_diff"].notnull()) - ] - calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] - if calculated_values.empty: - # Will only occur if all reported values are NaN when calculated values - # exist, or vice versa. - logger.warning( - "Warning: No calculated values have a corresponding reported value in the table." - ) - off_ratio = np.nan - else: - off_ratio = len(off_df) / len(calculated_values) - if off_ratio > calculation_tolerance: - raise AssertionError( - f"Calculations in {self.root_table} are off by {off_ratio:.2%}. Expected tolerance " - f"of {calculation_tolerance:.1%}." - ) - - # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0 or np.isnan(off_ratio): - logger.info( - f"{self.root_table}: has {len(off_df)} ({off_ratio:.02%}) records whose " - "calculations don't match. Adding correction records to make calculations " - "match reported values." - ) - corrections = off_df.copy() - - corrections[self.value_col] = ( - corrections[self.value_col].fillna(0.0) - - corrections["calculated_amount"] - ) - corrections["original_factoid"] = corrections["xbrl_factoid"] - corrections["xbrl_factoid"] = corrections["xbrl_factoid"] + "_correction" - corrections["row_type_xbrl"] = "correction" - corrections["is_within_table_calc"] = False - corrections["record_id"] = pd.NA - - calculated_df = pd.concat( - [calculated_df, corrections], axis="index" - ).reset_index(drop=True) - return calculated_df - def in_explosion_tables(table_name: str, in_explosion_table_names: list[str]) -> bool: """Determine if any of a list of table_names in the list of thre explosion tables. From d726e575b220591c8b308451a6f37e6478312e3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 20:41:09 +0000 Subject: [PATCH 03/20] [pre-commit.ci] auto fixes from pre-commit.com hooks For more information, see https://pre-commit.ci --- src/pudl/transform/ferc1.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 9d90533e13..170aa14a8d 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -24,16 +24,23 @@ from pydantic import validator import pudl -from pudl.analysis.classify_plants_ferc1 import (plants_steam_assign_plant_ids, - plants_steam_validate_ids) +from pudl.analysis.classify_plants_ferc1 import ( + plants_steam_assign_plant_ids, + plants_steam_validate_ids, +) from pudl.extract.ferc1 import TABLE_NAME_MAP_FERC1 from pudl.helpers import convert_cols_dtypes from pudl.metadata.fields import apply_pudl_dtypes from pudl.settings import Ferc1Settings -from pudl.transform.classes import (AbstractTableTransformer, InvalidRows, - RenameColumns, TableTransformParams, - TransformParams, cache_df, - enforce_snake_case) +from pudl.transform.classes import ( + AbstractTableTransformer, + InvalidRows, + RenameColumns, + TableTransformParams, + TransformParams, + cache_df, + enforce_snake_case, +) logger = pudl.logging_helpers.get_logger(__name__) From 936fc0fe417a58ed77c9f0372959cbee71068e31 Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Thu, 21 Sep 2023 15:13:25 -0600 Subject: [PATCH 04/20] add bool for adding the corrections --- src/pudl/output/ferc1.py | 1 + src/pudl/transform/ferc1.py | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index b083b04ce9..a399ec8d7c 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1478,6 +1478,7 @@ def generate_intertable_calculations( value_col=self.value_col, calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, table_name=self.root_table, + add_corrections=True, ) return calculated_df diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 9d90533e13..bdc4df167f 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -24,16 +24,23 @@ from pydantic import validator import pudl -from pudl.analysis.classify_plants_ferc1 import (plants_steam_assign_plant_ids, - plants_steam_validate_ids) +from pudl.analysis.classify_plants_ferc1 import ( + plants_steam_assign_plant_ids, + plants_steam_validate_ids, +) from pudl.extract.ferc1 import TABLE_NAME_MAP_FERC1 from pudl.helpers import convert_cols_dtypes from pudl.metadata.fields import apply_pudl_dtypes from pudl.settings import Ferc1Settings -from pudl.transform.classes import (AbstractTableTransformer, InvalidRows, - RenameColumns, TableTransformParams, - TransformParams, cache_df, - enforce_snake_case) +from pudl.transform.classes import ( + AbstractTableTransformer, + InvalidRows, + RenameColumns, + TableTransformParams, + TransformParams, + cache_df, + enforce_snake_case, +) logger = pudl.logging_helpers.get_logger(__name__) @@ -842,6 +849,7 @@ def reconcile_table_calculations( value_col=params.column_to_check, calculation_tolerance=params.calculation_tolerance, table_name=table_name, + add_corrections=True, ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) # Check that sub-total calculations sum to total. @@ -960,6 +968,7 @@ def check_calculcation_metrics( value_col: str, calculation_tolerance: float, table_name: str, + add_corrections: bool, ) -> pd.DataFrame: """Run the calculation metrics and determine if calculations are within tolerance.""" # Data types were very messy here, including pandas Float64 for the @@ -998,7 +1007,7 @@ def check_calculcation_metrics( ) # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0 or np.isnan(off_ratio): + if off_ratio > 0 or np.isnan(off_ratio) and add_corrections: logger.info( f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " "calculations don't match. Adding correction records to make calculations " From 31e334af27f85afa1e598bc32614a1ec418dbc5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 07:04:06 +0000 Subject: [PATCH 05/20] Update fsspec requirement from <2023.6.1,>=2021.7 to >=2021.7,<2023.9.3 Updates the requirements on [fsspec](https://github.com/fsspec/filesystem_spec) to permit the latest version. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2021.07.0...2023.9.2) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a96fceea01..143525de99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "dagster>=1.4,<1.5", # 1.2.2 is first version to support Python 3.11 "dask>=2021.8,<2023.9.2", "datapackage>=1.11,<1.16", # Transition datastore to use frictionless. - "fsspec>=2021.7,<2023.6.1", # For caching datastore on GCS + "fsspec>=2021.7,<2023.9.3", # For caching datastore on GCS "geopandas>=0.13,<0.15", "gcsfs>=2021.7,<2023.9.2", # For caching datastore on GCS "grpcio<1.56.0", From 4e5d60358b140f98b6a194c23253021f1c2b0634 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 07:04:48 +0000 Subject: [PATCH 06/20] Update dask requirement from <2023.9.2,>=2021.8 to >=2021.8,<2023.9.3 Updates the requirements on [dask](https://github.com/dask/dask) to permit the latest version. - [Changelog](https://github.com/dask/dask/blob/main/docs/release-procedure.md) - [Commits](https://github.com/dask/dask/compare/2021.08.0...2023.9.2) --- updated-dependencies: - dependency-name: dask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a96fceea01..311fb7b58f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "coloredlogs>=14.0,<15.1", # Dagster requires 14.0 "dagster-webserver>=1.4,<1.5", # 1.2.2 is first version to support Python 3.11 "dagster>=1.4,<1.5", # 1.2.2 is first version to support Python 3.11 - "dask>=2021.8,<2023.9.2", + "dask>=2021.8,<2023.9.3", "datapackage>=1.11,<1.16", # Transition datastore to use frictionless. "fsspec>=2021.7,<2023.6.1", # For caching datastore on GCS "geopandas>=0.13,<0.15", From 8de91025fa0328e332f79fd61d10a967ba54b65c Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 25 Sep 2023 13:35:46 -0600 Subject: [PATCH 07/20] Fix/add table names for a couple of tags. --- src/pudl/output/ferc1.py | 2 +- .../ferc1/xbrl_factoid_rate_base_tags.csv | 8 ++++---- src/pudl/transform/ferc1.py | 14 +++++++++----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index a399ec8d7c..7cfe12acdb 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1473,7 +1473,7 @@ def generate_intertable_calculations( calc_idx=calc_idx, value_col=self.value_col, ) - calculated_df = pudl.transform.ferc1.check_calculcation_metrics( + calculated_df = pudl.transform.ferc1.check_calculation_metrics( calculated_df=calculated_df, value_col=self.value_col, calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, diff --git a/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv b/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv index 0ea8932c91..705ec73e2f 100644 --- a/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv +++ b/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv @@ -1,7 +1,7 @@ xbrl_factoid_rmi,xbrl_factoid_rmi_renamed,xbrl_factoid,table_name,in_rate_base,utility_type,plant_function,plant_status,notes accrued_utility_revenues,accrued_utility_revenues,accrued_utility_revenues,balance_sheet_assets_ferc1,yes,,,,"At the option of the utility, the estimated amount accrued for service rendered, but not billed at the end of any accounting period, may be included herein. In case accruals are made for unbilled revenues, they shall be made likewise for unbilled expenses, such as for the purchase of energy." advances_for_gas,advances_for_gas,advances_for_gas,balance_sheet_assets_ferc1,yes,,,,"4/13/21 - changed from ""non-electric"" to ""mixed"" - could be for gas power plants." -allowances,allowances,allowances,electric_operating_expenses_ferc1,yes,,,, +allowances,allowances,allowance_inventory_and_withheld,balance_sheet_assets_ferc1,yes,,,, cash,cash,cash,balance_sheet_assets_ferc1,yes,,,, customer_accounts_receivable,customer_accounts_receivable,customer_accounts_receivable,balance_sheet_assets_ferc1,yes,,,,"service provided & billed/collected, asset earned on." derivative_instrument_assets,derivative_instrument_assets,derivative_instrument_assets,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? @@ -115,10 +115,10 @@ unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_su unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,no,,,, accounts_receivable_from_assoc_companies,accounts_receivable_from_assoc_companies,notes_receivable_from_associated_companies,balance_sheet_assets_ferc1,no,,,,Don't believe associated companies are involved in utility services? accum_prov_for_uncollectible_acct_credit,accum_prov_for_uncollectible_acct_credit,accumulated_provision_for_uncollectible_accounts_credit,,yes,,,,Losses on accounts receivable which may become collectible. -cash_and_working_funds_(non_major_only),cash_and_working_funds_(non_major_only),cash_and_working_funds,,yes,,,, -derivative_instrument_assets___hedges,derivative_instrument_assets___hedges,derivative_instrument_assets_hedges,,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +cash_and_working_funds_(non_major_only),cash_and_working_funds_(non_major_only),cash_and_working_funds,balance_sheet_assets_ferc1,yes,,,, +derivative_instrument_assets___hedges,derivative_instrument_assets___hedges,derivative_instrument_assets_hedges,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? gas_stored_underground___current,gas_stored_underground___current,gas_stored_current,balance_sheet_assets_ferc1,yes,,,, -long_term_portion_of_derivative_instrument_assets,long_term_portion_of_derivative_instrument_assets,derivative_instrument_assets_hedges_long_term,,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +long_term_portion_of_derivative_instrument_assets,long_term_portion_of_derivative_instrument_assets,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? long_term_portion_of_derivative_instrument_assets___hedges,long_term_portion_of_derivative_instrument_assets___hedges,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? residuals_(elec)_and_extracted_products,residuals_(elec)_and_extracted_products,residuals,balance_sheet_assets_ferc1,yes,,,, working_fund,working_fund,working_funds,balance_sheet_assets_ferc1,yes,,,, diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index bdc4df167f..7438929999 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -752,6 +752,7 @@ def reconcile_table_calculations( xbrl_factoid_name: str, table_name: str, params: ReconcileTableCalculations, + add_corrections: bool = True, ) -> pd.DataFrame: """Ensure intra-table calculated values match reported values within a tolerance. @@ -773,6 +774,8 @@ def reconcile_table_calculations( xbrl_factoid_name: column name of the XBRL factoid in the processed table. table_name: name of the PUDL table. params: :class:`ReconcileTableCalculations` parameters. + add_corrections: Whether or not to create _correction records that force all + calculations to add up correctly. """ # If we don't have this value, we aren't doing any calculation checking: if params.column_to_check is None or calculation_components.empty: @@ -844,12 +847,12 @@ def reconcile_table_calculations( calc_idx=calc_idx, value_col=params.column_to_check, ) - calculated_df = check_calculcation_metrics( + calculated_df = check_calculation_metrics( calculated_df=calculated_df, value_col=params.column_to_check, calculation_tolerance=params.calculation_tolerance, table_name=table_name, - add_corrections=True, + add_corrections=add_corrections, ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) # Check that sub-total calculations sum to total. @@ -963,12 +966,12 @@ def calculate_values_from_components( return calculated_df -def check_calculcation_metrics( +def check_calculation_metrics( calculated_df: pd.DataFrame, value_col: str, calculation_tolerance: float, table_name: str, - add_corrections: bool, + add_corrections: bool = True, ) -> pd.DataFrame: """Run the calculation metrics and determine if calculations are within tolerance.""" # Data types were very messy here, including pandas Float64 for the @@ -1007,7 +1010,7 @@ def check_calculcation_metrics( ) # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0 or np.isnan(off_ratio) and add_corrections: + if (off_ratio > 0 or np.isnan(off_ratio)) and add_corrections: logger.info( f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " "calculations don't match. Adding correction records to make calculations " @@ -2393,6 +2396,7 @@ def reconcile_table_calculations( xbrl_factoid_name=self.params.xbrl_factoid_name, table_name=self.table_id.value, params=params, + add_corrections=False, ) return df From c928f4fddd79064a2ee69a142044382ac275885b Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 25 Sep 2023 13:51:41 -0600 Subject: [PATCH 08/20] Remove obsolete check for multi-valued calculation weights. --- src/pudl/output/ferc1.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 7cfe12acdb..0ceaac365c 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1558,21 +1558,6 @@ def unique_associations(cls, v: pd.DataFrame, values) -> pd.DataFrame: assert not v.duplicated(subset=pks, keep=False).any() return v - @validator("exploded_calcs") - def single_valued_weights(cls, v: pd.DataFrame, values) -> pd.DataFrame: - """Ensure that every calculation component has a uniquely specified weight.""" - multi_valued_weights = ( - v.groupby(values["calc_cols"], dropna=False)["weight"] - .transform("nunique") - .gt(1) - ) - if multi_valued_weights.any(): - logger.warning( - f"Found {sum(multi_valued_weights)} calculations with conflicting " - "weights." - ) - return v - @validator("exploded_calcs") def calcs_have_required_cols(cls, v: pd.DataFrame, values) -> pd.DataFrame: """Ensure exploded calculations include all required columns.""" From 52c56bb0ac0eb74fb8d7c2db8aa15436f629df88 Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 25 Sep 2023 14:28:10 -0600 Subject: [PATCH 09/20] Turn corrections back on --- src/pudl/transform/ferc1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 7438929999..2bfae0ff5b 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -2396,7 +2396,7 @@ def reconcile_table_calculations( xbrl_factoid_name=self.params.xbrl_factoid_name, table_name=self.table_id.value, params=params, - add_corrections=False, + add_corrections=True, ) return df From 1e31ebbabefb997a01bf08e98d07f4710bff837a Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 25 Sep 2023 15:38:53 -0600 Subject: [PATCH 10/20] Require one_to_many validation in all tables when reconciling errors. --- src/pudl/transform/ferc1.py | 4 +--- test/unit/transform/ferc1_test.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 2bfae0ff5b..a0dde28010 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -843,7 +843,6 @@ def reconcile_table_calculations( calculated_df = calculate_values_from_components( data=df, calculation_components=intra_tbl_calcs, - validate="one_to_many", calc_idx=calc_idx, value_col=params.column_to_check, ) @@ -894,7 +893,6 @@ def reconcile_table_calculations( def calculate_values_from_components( calculation_components: pd.DataFrame, data: pd.DataFrame, - validate: Literal["one_to_many", "many_to_many"], calc_idx: list[str], value_col: str, ) -> pd.DataFrame: @@ -936,7 +934,7 @@ def calculate_values_from_components( pd.merge( calculation_components, data, - validate=validate, + validate="one_to_many", on=calc_idx, ) # apply the weight from the calc to convey the sign before summing. diff --git a/test/unit/transform/ferc1_test.py b/test/unit/transform/ferc1_test.py index b366a4ceb4..9dde46fd10 100644 --- a/test/unit/transform/ferc1_test.py +++ b/test/unit/transform/ferc1_test.py @@ -451,7 +451,6 @@ def test_calculate_values_from_components(): out_ksr = calculate_values_from_components( calculation_components=calculation_components_ksr, data=data_ksr, - validate="one_to_many", calc_idx=["table_name", "xbrl_factoid", "planet"], value_col="value", ) From 48190e9b8ca362a539d6502276463ce43634c10c Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Sat, 23 Sep 2023 08:35:22 -0600 Subject: [PATCH 11/20] standardize the calc checks for the total to subtotal calcs --- src/pudl/output/ferc1.py | 29 ++++++++++-- src/pudl/transform/ferc1.py | 77 ++++++++++++++++--------------- test/unit/transform/ferc1_test.py | 55 +++++++++++----------- 3 files changed, 94 insertions(+), 67 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 0ceaac365c..4f3da74869 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1170,7 +1170,12 @@ def exploded_calcs(self: Self): # things for legibility. calc_explode = ( calc_explode[calc_explode.is_in_explosion] - .loc[:, parent_cols + calc_cols + ["weight", "is_within_table_calc"]] + .loc[ + :, + parent_cols + + calc_cols + + ["weight", "is_within_table_calc", "is_total_to_subdimensions_calc"], + ] .drop_duplicates() .set_index(parent_cols + calc_cols) .sort_index() @@ -1466,10 +1471,12 @@ def generate_intertable_calculations( f"{list(calculations_intertable.xbrl_factoid.unique())}." ) calc_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] + logger.info("Checking inter-table, non-total to subtotal calcs.") calculated_df = pudl.transform.ferc1.calculate_values_from_components( - calculation_components=calculations_intertable, + calculation_components=calculations_intertable[ + ~calculations_intertable.is_total_to_subdimensions_calc + ], data=exploded, - validate="one_to_many", calc_idx=calc_idx, value_col=self.value_col, ) @@ -1480,6 +1487,22 @@ def generate_intertable_calculations( table_name=self.root_table, add_corrections=True, ) + logger.info("Checking sub-total calcs.") + subtotal_calcs = pudl.transform.ferc1.calculate_values_from_components( + calculation_components=calculations_intertable[ + calculations_intertable.is_total_to_subdimensions_calc + ], + data=exploded, + calc_idx=calc_idx, + value_col=self.value_col, + ) + subtotal_calcs = pudl.transform.ferc1.check_calculcation_metrics( + calculated_df=subtotal_calcs, + value_col=self.value_col, + calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, + table_name=self.root_table, + add_corrections=True, + ) return calculated_df diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 2bfae0ff5b..9001164613 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -749,6 +749,7 @@ class ReconcileTableCalculations(TransformParams): def reconcile_table_calculations( df: pd.DataFrame, calculation_components: pd.DataFrame, + xbrl_metadata: pd.DataFrame, xbrl_factoid_name: str, table_name: str, params: ReconcileTableCalculations, @@ -803,7 +804,9 @@ def reconcile_table_calculations( calc_idx = ["xbrl_factoid", "table_name"] + dim_cols if dim_cols: - table_dims = df[calc_idx].drop_duplicates(keep="first") + table_dims = ( + df[calc_idx].drop_duplicates(keep="first").assign(table_name=table_name) + ) # need to add in the correction dimensions. they don't show up in the data at # this point so we don't have the dimensions yet. NOTE: this could have been # done by adding the dims into table_dims..... maybe would have been more @@ -839,11 +842,9 @@ def reconcile_table_calculations( & intra_tbl_calcs[f"{dim}_parent"].isnull() ) ] - pks = pudl.metadata.classes.Resource.from_id(table_name).schema.primary_key calculated_df = calculate_values_from_components( data=df, calculation_components=intra_tbl_calcs, - validate="one_to_many", calc_idx=calc_idx, value_col=params.column_to_check, ) @@ -857,36 +858,37 @@ def reconcile_table_calculations( # Check that sub-total calculations sum to total. if params.subtotal_column is not None: - sub_group_col = params.subtotal_column - pks_wo_subgroup = [col for col in pks if col != sub_group_col] - calculated_df["sub_total_sum"] = ( - calculated_df.pipe(lambda df: df[df[sub_group_col] != "total"]) - .groupby(pks_wo_subgroup)[params.column_to_check] - .transform("sum") # For each group, calculate sum of sub-components - ) - calculated_df["sub_total_sum"] = calculated_df["sub_total_sum"].fillna( - calculated_df[params.column_to_check] # Fill in value from 'total' column - ) - sub_total_errors = ( - calculated_df.groupby(pks_wo_subgroup) - # If subcomponent sum != total sum, we have nunique()>1 - .filter(lambda x: x["sub_total_sum"].nunique() > 1).groupby( # noqa: PD101 - pks_wo_subgroup - ) + logger.info( + f"Checking total-to-subtotal calculations within {params.subtotal_column}" ) - off_ratio_sub = ( - sub_total_errors.ngroups / calculated_df.groupby(pks_wo_subgroup).ngroups + meta_w_dims = xbrl_metadata.assign( + **{dim: pd.NA for dim in dim_cols} | {"table_name": table_name} + ).pipe( + make_calculation_dimensions_explicit, + table_dimensions_ferc1=table_dims, + dimensions=dim_cols, ) - if sub_total_errors.ngroups > 0: - logger.warning( - f"{table_name}: has {sub_total_errors.ngroups} ({off_ratio_sub:.02%}) sub-total calculations that don't " - "sum to the equivalent total column." - ) - if off_ratio_sub > params.subtotal_calculation_tolerance: - raise AssertionError( - f"Sub-total calculations in {table_name} are off by {off_ratio_sub}. Expected tolerance " - f"of {params.subtotal_calculation_tolerance}." - ) + calc_comps_w_totals = infer_intra_factoid_totals( + intra_tbl_calcs, + meta_w_dims=meta_w_dims, + table_dimensions=table_dims, + dimensions=dim_cols, + ) + subtotal_calcs = calculate_values_from_components( + data=df, + calculation_components=calc_comps_w_totals[ + calc_comps_w_totals.is_total_to_subdimensions_calc + ], + calc_idx=calc_idx, + value_col=params.column_to_check, + ) + subtotal_calcs = check_calculcation_metrics( + calculated_df=subtotal_calcs, + value_col=params.column_to_check, + calculation_tolerance=params.calculation_tolerance, + table_name=table_name, + add_corrections=True, + ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) return calculated_df @@ -894,7 +896,6 @@ def reconcile_table_calculations( def calculate_values_from_components( calculation_components: pd.DataFrame, data: pd.DataFrame, - validate: Literal["one_to_many", "many_to_many"], calc_idx: list[str], value_col: str, ) -> pd.DataFrame: @@ -908,8 +909,6 @@ def calculate_values_from_components( data: exploded FERC data to apply the calculations to. Primary key should be ``report_year``, ``utility_id_ferc1``, ``table_name``, ``xbrl_factoid``, and whatever additional dimensions are relevant to the data. - validate: type of merge validation to apply when initially merging the calculation - components (left) and the data (right). calc_idx: primary key columns that uniquely identify a calculation component (not including the ``_parent`` columns). value_col: label of the column in ``data`` that contains the values to apply the @@ -936,7 +935,7 @@ def calculate_values_from_components( pd.merge( calculation_components, data, - validate=validate, + validate="one_to_many", on=calc_idx, ) # apply the weight from the calc to convey the sign before summing. @@ -2394,6 +2393,7 @@ def reconcile_table_calculations( df=df, calculation_components=self.xbrl_calculations, xbrl_factoid_name=self.params.xbrl_factoid_name, + xbrl_metadata=self.xbrl_metadata, table_name=self.table_id.value, params=params, add_corrections=True, @@ -6040,7 +6040,12 @@ def infer_intra_factoid_totals( lambda dim: f"{dim}_parent", axis="columns" ) inferred_totals = inferred_totals.fillna(child_values) - calcs_with_totals = pd.concat([calc_components, inferred_totals]) + calcs_with_totals = pd.concat( + [ + calc_components.assign(is_total_to_subdimensions_calc=False), + inferred_totals.assign(is_total_to_subdimensions_calc=True), + ] + ) # verification + deduping below. diff --git a/test/unit/transform/ferc1_test.py b/test/unit/transform/ferc1_test.py index b366a4ceb4..a942c436c3 100644 --- a/test/unit/transform/ferc1_test.py +++ b/test/unit/transform/ferc1_test.py @@ -451,7 +451,6 @@ def test_calculate_values_from_components(): out_ksr = calculate_values_from_components( calculation_components=calculation_components_ksr, data=data_ksr, - validate="one_to_many", calc_idx=["table_name", "xbrl_factoid", "planet"], value_col="value", ) @@ -663,20 +662,20 @@ def test_adding_parent_dimensions(): pd.read_csv( StringIO( """ -table_name_parent,xbrl_factoid_parent,dim_x_parent,dim_y_parent,table_name,xbrl_factoid,dim_x,dim_y,is_within_table_calc,weight -table_a,fact_1,voyager,coffee,table_a,fact_3,voyager,coffee,True,2 -table_a,fact_1,voyager,in,table_a,fact_3,voyager,in,True,2 -table_a,fact_1,voyager,that,table_a,fact_3,voyager,that,True,2 -table_a,fact_1,voyager,nebula,table_a,fact_3,voyager,nebula,True,2 -table_a,fact_1,voyager,total,table_a,fact_3,voyager,total,True,2 -table_a,fact_1,voyager,total,table_a,fact_1,voyager,coffee,True,1 -table_a,fact_1,voyager,total,table_a,fact_1,voyager,in,True,1 -table_a,fact_1,voyager,total,table_a,fact_1,voyager,that,True,1 -table_a,fact_1,voyager,total,table_a,fact_1,voyager,nebula,True,1 -table_a,fact_3,voyager,total,table_a,fact_3,voyager,coffee,True,1 -table_a,fact_3,voyager,total,table_a,fact_3,voyager,in,True,1 -table_a,fact_3,voyager,total,table_a,fact_3,voyager,that,True,1 -table_a,fact_3,voyager,total,table_a,fact_3,voyager,nebula,True,1 +table_name_parent,xbrl_factoid_parent,dim_x_parent,dim_y_parent,table_name,xbrl_factoid,dim_x,dim_y,is_within_table_calc,weight,is_total_to_subdimensions_calc +table_a,fact_1,voyager,coffee,table_a,fact_3,voyager,coffee,True,2,False +table_a,fact_1,voyager,in,table_a,fact_3,voyager,in,True,2,False +table_a,fact_1,voyager,that,table_a,fact_3,voyager,that,True,2,False +table_a,fact_1,voyager,nebula,table_a,fact_3,voyager,nebula,True,2,False +table_a,fact_1,voyager,total,table_a,fact_3,voyager,total,True,2,False +table_a,fact_1,voyager,total,table_a,fact_1,voyager,coffee,True,1,True +table_a,fact_1,voyager,total,table_a,fact_1,voyager,in,True,1,True +table_a,fact_1,voyager,total,table_a,fact_1,voyager,that,True,1,True +table_a,fact_1,voyager,total,table_a,fact_1,voyager,nebula,True,1,True +table_a,fact_3,voyager,total,table_a,fact_3,voyager,coffee,True,1,True +table_a,fact_3,voyager,total,table_a,fact_3,voyager,in,True,1,True +table_a,fact_3,voyager,total,table_a,fact_3,voyager,that,True,1,True +table_a,fact_3,voyager,total,table_a,fact_3,voyager,nebula,True,1,True """ ) ) @@ -790,19 +789,19 @@ def test_multi_dims_totals(): pd.read_csv( StringIO( """ -table_name_parent,xbrl_factoid_parent,utility_type_parent,plant_status_parent,plant_function_parent,table_name,xbrl_factoid,utility_type,plant_status,plant_function,is_within_table_calc,weight -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1 -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1 +table_name_parent,xbrl_factoid_parent,utility_type_parent,plant_status_parent,plant_function_parent,table_name,xbrl_factoid,utility_type,plant_status,plant_function,is_within_table_calc,weight,is_total_to_subdimensions_calc +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1,True +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1,True """ ) ) From 31e3a5447d3f5f6cbc9a0143e8b194e1513608bc Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Sat, 23 Sep 2023 08:46:23 -0600 Subject: [PATCH 12/20] lil name cleanup --- src/pudl/output/ferc1.py | 6 +++--- src/pudl/transform/ferc1.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 4f3da74869..b3a030c3b1 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1373,7 +1373,7 @@ def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame """ exploded = ( self.initial_explosion_concatenation(tables_to_explode) - .pipe(self.generate_intertable_calculations) + .pipe(self.reconcile_intertable_calculations) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... @@ -1447,7 +1447,7 @@ def initial_explosion_concatenation( ) return exploded - def generate_intertable_calculations( + def reconcile_intertable_calculations( self: Self, exploded: pd.DataFrame ) -> pd.DataFrame: """Generate calculated values for inter-table calculated factoids. @@ -1496,7 +1496,7 @@ def generate_intertable_calculations( calc_idx=calc_idx, value_col=self.value_col, ) - subtotal_calcs = pudl.transform.ferc1.check_calculcation_metrics( + subtotal_calcs = pudl.transform.ferc1.check_calculation_metrics( calculated_df=subtotal_calcs, value_col=self.value_col, calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 9001164613..1cd60a7638 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -882,7 +882,7 @@ def reconcile_table_calculations( calc_idx=calc_idx, value_col=params.column_to_check, ) - subtotal_calcs = check_calculcation_metrics( + subtotal_calcs = check_calculation_metrics( calculated_df=subtotal_calcs, value_col=params.column_to_check, calculation_tolerance=params.calculation_tolerance, From 82ee9db6f29863a3e28419bc60e79605aa971607 Mon Sep 17 00:00:00 2001 From: thinky Date: Mon, 25 Sep 2023 12:17:05 -0400 Subject: [PATCH 13/20] Fix inter-table 1:1 linkages --- .../xbrl_calculation_component_fixes.csv | 180 +++++++++--------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv index b68db70768..8d9ee3896a 100644 --- a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv +++ b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv @@ -1,42 +1,42 @@ table_name_parent,xbrl_factoid_parent,table_name,xbrl_factoid,weight,utility_type,plant_function,plant_status balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,abandonment_of_leases,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1.0,,, +balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1,total,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_other_utility_plant_utility_plant_in_service,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_plant_acquisition_adjustment,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_leased_to_others,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_and_amortization_utility_plant_held_for_future_use,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_utility_plant_in_service,,,, -balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,1.0,,, +balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,1,total,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_hedges_long_term,,,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_long_term,,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_hedges_long_term,-1.0,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_long_term,-1.0,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_noncurrent_portion_of_allowances,-1.0,,, +balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_hedges_long_term,-1,,, +balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_long_term,-1,,, +balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_noncurrent_portion_of_allowances,-1,,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,noncurrent_portion_of_allowances,,,, -balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_survey_and_investigation_charges,1.0,,, -balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_and_other_survey_and_investigation_charges,1.0,,, -balance_sheet_assets_ferc1,nuclear_fuel_net,balance_sheet_assets_ferc1,nuclear_fuel,1.0,,, +balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_survey_and_investigation_charges,1,,, +balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_and_other_survey_and_investigation_charges,1,,, +balance_sheet_assets_ferc1,nuclear_fuel_net,balance_sheet_assets_ferc1,nuclear_fuel,1,,, balance_sheet_assets_ferc1,nuclear_fuel_net,nuclear_fuel_materials_ferc1,nuclear_fuel_materials_and_assemblies,,,, balance_sheet_assets_ferc1,nuclear_fuel_net,nuclear_fuel_materials_ferc1,spent_nuclear_fuel,,,, -balance_sheet_assets_ferc1,other_property_and_investments,balance_sheet_assets_ferc1,special_funds_all,1.0,,, +balance_sheet_assets_ferc1,other_property_and_investments,balance_sheet_assets_ferc1,special_funds_all,1,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,construction_work_in_progress,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,utility_plant,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_acquisition_adjustment,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,1.0,total,, +balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,1,total,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_held_for_future_use,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_leased_to_others,,,, balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,,,, balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,,,, -balance_sheet_assets_ferc1,utility_plant_net,utility_plant_summary_ferc1,utility_plant_net,1.0,total,, -balance_sheet_liabilities_ferc1,deferred_credits,balance_sheet_liabilities_ferc1,accumulated_deferred_income_taxes,1.0,,, -balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,1.0,,, +balance_sheet_assets_ferc1,utility_plant_net,utility_plant_summary_ferc1,utility_plant_net,1,total,, +balance_sheet_liabilities_ferc1,deferred_credits,balance_sheet_liabilities_ferc1,accumulated_deferred_income_taxes,1,,, +balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,1,total,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities,,,, -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities,-1.0,,, +balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities,-1,,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities_hedges,,,, -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities_hedges,-1.0,,, -electric_energy_sources_ferc1,sources_of_energy,electric_energy_sources_ferc1,megawatt_hours_purchased,1.0,,, +balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities_hedges,-1,,, +electric_energy_sources_ferc1,sources_of_energy,electric_energy_sources_ferc1,megawatt_hours_purchased,1,,, electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,electric_expenses_hydraulic_power_generation,,,, electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,hydraulic_expenses,,,, electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_of_electric_plant_hydraulic_power_generation,,,, @@ -67,82 +67,82 @@ electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_hydro_ferc1,opex_operations,,,, electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_pumped_storage_ferc1,opex_operations,,,, electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_steam_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,transmission_operation_expense,electric_operating_expenses_ferc1,load_dispatching_transmission_expense,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,forfeited_discounts,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,interdepartmental_rents,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_revenue,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_service_revenues,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_electric_revenue,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_miscellaneous_operating_revenues,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,regional_transmission_service_revenues,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,rent_from_electric_property,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,revenues_from_transmission_of_electricity_of_others,1.0,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,sales_of_water_and_water_power,1.0,,, -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,large_or_industrial,1.0,,, -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,small_or_commercial,1.0,,, +electric_operating_expenses_ferc1,transmission_operation_expense,electric_operating_expenses_ferc1,load_dispatching_transmission_expense,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,forfeited_discounts,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,interdepartmental_rents,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_revenue,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_service_revenues,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_electric_revenue,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_miscellaneous_operating_revenues,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,regional_transmission_service_revenues,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,rent_from_electric_property,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,revenues_from_transmission_of_electricity_of_others,1,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,sales_of_water_and_water_power,1,,, +electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,large_or_industrial,1,,, +electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,small_or_commercial,1,,, electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electricity_sales_by_rate_schedule_ferc1,commercial_and_industrial,,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,book_cost_of_asset_retirement_costs,1.0,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,depreciation_provision,1.0,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,net_charges_for_retired_plant,1.0,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,other_adjustments_to_accumulated_depreciation,1.0,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,starting_balance,1.0,,, -income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_limited_term_electric_plant,1.0,electric,total, -income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_other_electric_plant,1.0,electric,total, -income_statement_ferc1,depreciation_expense,depreciation_amortization_summary_ferc1,depreciation_expense,1.0,electric,total, -income_statement_ferc1,depreciation_expense_for_asset_retirement_costs,depreciation_amortization_summary_ferc1,depreciation_expense_asset_retirement,1.0,electric,total, -income_statement_ferc1,income_before_extraordinary_items,income_statement_ferc1,net_utility_operating_income,1.0,,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,distribution_maintenance_expense_electric,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_maintenance_expense,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,maintenance_of_general_plant,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,nuclear_power_generation_maintenance_expense,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,other_power_generation_maintenance_expense,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,regional_market_maintenance_expense,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,steam_power_generation_maintenance_expense,1.0,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,transmission_maintenance_expense_electric,1.0,electric,, -income_statement_ferc1,operating_revenues,electric_operating_revenues_ferc1,electric_operating_revenues,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,administrative_and_general_operation_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_account_expenses,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_service_and_information_expenses,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,distribution_operation_expenses_electric,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_operations_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,nuclear_power_generation_operations_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,other_power_generation_operations_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,regional_market_operation_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,sales_expenses,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,steam_power_generation_operations_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,transmission_operation_expense,1.0,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,power_production_expenses,1.0,electric,, -income_statement_ferc1,other_income_deductions,income_statement_ferc1,miscellaneous_deductions,1.0,,, -income_statement_ferc1,taxes_on_other_income_and_deductions,income_statement_ferc1,investment_tax_credits,-1.0,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,distribution_plant,1.0,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,general_plant,1.0,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,intangible_plant,1.0,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,production_plant,1.0,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,1.0,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_plant,1.0,,, -retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings,1.0,,, -retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings_amortization_reserve_federal,1.0,,, -retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,1.0,,, -retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_credit,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_debit,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,appropriations_of_retained_earnings,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,balance_transferred_from_income,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_common_stock,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_preferred_stock,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,transfers_from_unappropriated_undistributed_subsidiary_earnings,1.0,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings_previous_year,1.0,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,changes_unappropriated_undistributed_subsidiary_earnings_credits,1.0,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,dividends_received,-1.0,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,equity_in_earnings_of_subsidiary_companies,1.0,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings_previous_year,1.0,,, -balance_sheet_liabilities_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,1.0,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,book_cost_of_asset_retirement_costs,1,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,depreciation_provision,1,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,net_charges_for_retired_plant,1,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,other_adjustments_to_accumulated_depreciation,1,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,starting_balance,1,,, +income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_limited_term_electric_plant,1,electric,total, +income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_other_electric_plant,1,electric,total, +income_statement_ferc1,depreciation_expense,depreciation_amortization_summary_ferc1,depreciation_expense,1,electric,total, +income_statement_ferc1,depreciation_expense_for_asset_retirement_costs,depreciation_amortization_summary_ferc1,depreciation_expense_asset_retirement,1,electric,total, +income_statement_ferc1,income_before_extraordinary_items,income_statement_ferc1,net_utility_operating_income,1,,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,distribution_maintenance_expense_electric,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_maintenance_expense,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,maintenance_of_general_plant,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,nuclear_power_generation_maintenance_expense,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,other_power_generation_maintenance_expense,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,regional_market_maintenance_expense,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,steam_power_generation_maintenance_expense,1,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,transmission_maintenance_expense_electric,1,electric,, +income_statement_ferc1,operating_revenues,electric_operating_revenues_ferc1,electric_operating_revenues,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,administrative_and_general_operation_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_account_expenses,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_service_and_information_expenses,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,distribution_operation_expenses_electric,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_operations_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,nuclear_power_generation_operations_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,other_power_generation_operations_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,regional_market_operation_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,sales_expenses,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,steam_power_generation_operations_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,transmission_operation_expense,1,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,power_production_expenses,1,electric,, +income_statement_ferc1,other_income_deductions,income_statement_ferc1,miscellaneous_deductions,1,,, +income_statement_ferc1,taxes_on_other_income_and_deductions,income_statement_ferc1,investment_tax_credits,-1,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,distribution_plant,1,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,general_plant,1,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,intangible_plant,1,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,production_plant,1,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,1,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_plant,1,,, +retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings,1,,, +retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings_amortization_reserve_federal,1,,, +retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,1,,, +retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_credit,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_debit,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,appropriations_of_retained_earnings,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,balance_transferred_from_income,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_common_stock,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_preferred_stock,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,transfers_from_unappropriated_undistributed_subsidiary_earnings,1,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings_previous_year,1,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,changes_unappropriated_undistributed_subsidiary_earnings_credits,1,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,dividends_received,-1,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,equity_in_earnings_of_subsidiary_companies,1,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings_previous_year,1,,, +balance_sheet_liabilities_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,1,,, utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, -utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,1.0,electric,total,total +utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,1,electric,total,in_service utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified,,,, -utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_property_under_capital_leases,1.0,,, +utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_property_under_capital_leases,1,,, utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_property_under_capital_leases,,,, -utility_plant_summary_ferc1,utility_plant_in_service_experimental_plant_unclassified,plant_in_service_ferc1,experimental_electric_plant_unclassified,1.0,electric,, -utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_purchased,1.0,electric,, -utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_sold,-1.0,electric,, +utility_plant_summary_ferc1,utility_plant_in_service_experimental_plant_unclassified,plant_in_service_ferc1,experimental_electric_plant_unclassified,1,electric,, +utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_purchased,1,electric,, +utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_sold,-1,electric,, utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,utility_plant,,,, From 246709752556b0385125e96ac377a0275d7c1685 Mon Sep 17 00:00:00 2001 From: thinky Date: Mon, 25 Sep 2023 17:34:40 -0400 Subject: [PATCH 14/20] Remove commits from other branch --- src/pudl/output/ferc1.py | 187 +++++++++++++++++++++++------- src/pudl/transform/ferc1.py | 122 ++++++++++++------- test/unit/transform/ferc1_test.py | 55 ++++----- 3 files changed, 256 insertions(+), 108 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index b3a030c3b1..54801ec5a1 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1170,12 +1170,7 @@ def exploded_calcs(self: Self): # things for legibility. calc_explode = ( calc_explode[calc_explode.is_in_explosion] - .loc[ - :, - parent_cols - + calc_cols - + ["weight", "is_within_table_calc", "is_total_to_subdimensions_calc"], - ] + .loc[:, parent_cols + calc_cols + ["weight", "is_within_table_calc"]] .drop_duplicates() .set_index(parent_cols + calc_cols) .sort_index() @@ -1373,7 +1368,11 @@ def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame """ exploded = ( self.initial_explosion_concatenation(tables_to_explode) - .pipe(self.reconcile_intertable_calculations) + .pipe(self.generate_intertable_calculations) + .pipe( + self.reconcile_intertable_calculations, + self.calculation_tolerance.intertable_calculation_errors, + ) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... @@ -1447,7 +1446,7 @@ def initial_explosion_concatenation( ) return exploded - def reconcile_intertable_calculations( + def generate_intertable_calculations( self: Self, exploded: pd.DataFrame ) -> pd.DataFrame: """Generate calculated values for inter-table calculated factoids. @@ -1470,39 +1469,147 @@ def reconcile_intertable_calculations( f"{self.root_table}: Reconcile inter-table calculations: " f"{list(calculations_intertable.xbrl_factoid.unique())}." ) - calc_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] - logger.info("Checking inter-table, non-total to subtotal calcs.") - calculated_df = pudl.transform.ferc1.calculate_values_from_components( - calculation_components=calculations_intertable[ - ~calculations_intertable.is_total_to_subdimensions_calc - ], - data=exploded, - calc_idx=calc_idx, - value_col=self.value_col, - ) - calculated_df = pudl.transform.ferc1.check_calculation_metrics( - calculated_df=calculated_df, - value_col=self.value_col, - calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, - table_name=self.root_table, - add_corrections=True, - ) - logger.info("Checking sub-total calcs.") - subtotal_calcs = pudl.transform.ferc1.calculate_values_from_components( - calculation_components=calculations_intertable[ - calculations_intertable.is_total_to_subdimensions_calc - ], - data=exploded, - calc_idx=calc_idx, - value_col=self.value_col, - ) - subtotal_calcs = pudl.transform.ferc1.check_calculation_metrics( - calculated_df=subtotal_calcs, - value_col=self.value_col, - calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, - table_name=self.root_table, - add_corrections=True, + # compile the lists of columns we are going to use later + calc_component_idx = ["table_name", "xbrl_factoid"] + self.other_dimensions + # Merge the reported data and the calculation component metadata to enable + # validation of calculated values. Here the data table exploded is supplying the + # values associated with individual calculation components, and the table_name + # and xbrl_factoid to which we aggregate are coming from the calculation + # components table. After merging we use the weights to adjust the reported + # values so they can be summed directly. This gives us aggregated calculated + # values that can later be compared to the higher level reported values. + + # the validation is one_many in all instances expect for the xbrl_factoid + # construction_work_in_progress in the balance_sheet_assets_ferc1 explosion. + # this may be a problem in the calculations that we should track down in #2717 + validate = ( + "one_to_many" + if self.root_table != "balance_sheet_assets_ferc1" + else "many_to_many" + ) + # we are going to merge the data onto the calc components with the _parent + # column names, so the groupby after the merge needs a set of by cols with the + # _parent suffix + meta_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] + gby_parent = [ + f"{col}_parent" if col in meta_idx else col for col in self.exploded_pks + ] + calc_df = ( + pd.merge( + calculations_intertable, + exploded, + validate=validate, + on=calc_component_idx, + ) + # apply the weight from the calc to convey the sign before summing. + .assign(calculated_amount=lambda x: x[self.value_col] * x.weight) + .groupby(gby_parent, as_index=False, dropna=False)[["calculated_amount"]] + .sum(min_count=1) + ) + # remove the _parent suffix so we can merge these calculated values back onto + # the data using the original pks + calc_df.columns = calc_df.columns.str.removesuffix("_parent") + calculated_df = pd.merge( + exploded, + calc_df, + on=self.exploded_pks, + how="outer", + validate="1:1", + indicator=True, + ) + + assert calculated_df[ + (calculated_df._merge == "right_only") + & (calculated_df[self.value_col].notnull()) + ].empty + + calculated_df = calculated_df.drop(columns=["_merge"]) + # Force value_col to be a float to prevent any hijinks with calculating differences. + calculated_df[self.value_col] = calculated_df[self.value_col].astype(float) + + return calculated_df + + def reconcile_intertable_calculations( + self: Self, calculated_df: pd.DataFrame, calculation_tolerance: float = 0.05 + ): + """Ensure inter-table calculated values match reported values within a tolerance. + + In addition to checking whether all reported "calculated" values match the output + of our repaired calculations, this function adds a correction record to the + dataframe that is included in the calculations so that after the fact the + calculations match exactly. This is only done when the fraction of records that + don't match within the tolerances of :func:`numpy.isclose` is below a set + threshold. + + Note that only calculations which are off by a significant amount result in the + creation of a correction record. Many calculations are off from the reported values + by exaclty one dollar, presumably due to rounding errrors. These records typically + do not fail the :func:`numpy.isclose()` test and so are not corrected. + + Args: + calculated_df: table with calculated fields + calculation_tolerance: What proportion (0-1) of calculated values are + allowed to be incorrect without raising an AssertionError. + """ + if "calculated_amount" not in calculated_df.columns: + return calculated_df + + # Data types were very messy here, including pandas Float64 for the + # calculated_amount columns which did not work with the np.isclose(). Not sure + # why these are cropping up. + calculated_df = calculated_df.convert_dtypes(convert_floating=False).astype( + {self.value_col: "float64", "calculated_amount": "float64"} + ) + calculated_df = calculated_df.assign( + abs_diff=lambda x: abs(x[self.value_col] - x.calculated_amount), + rel_diff=lambda x: np.where( + (x[self.value_col] != 0.0), + abs(x.abs_diff / x[self.value_col]), + np.nan, + ), ) + off_df = calculated_df[ + ~np.isclose(calculated_df.calculated_amount, calculated_df[self.value_col]) + & (calculated_df["abs_diff"].notnull()) + ] + calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] + if calculated_values.empty: + # Will only occur if all reported values are NaN when calculated values + # exist, or vice versa. + logger.warning( + "Warning: No calculated values have a corresponding reported value in the table." + ) + off_ratio = np.nan + else: + off_ratio = len(off_df) / len(calculated_values) + if off_ratio > calculation_tolerance: + raise AssertionError( + f"Calculations in {self.root_table} are off by {off_ratio:.2%}. Expected tolerance " + f"of {calculation_tolerance:.1%}." + ) + + # # We'll only get here if the proportion of calculations that are off is acceptable + if off_ratio > 0 or np.isnan(off_ratio): + logger.info( + f"{self.root_table}: has {len(off_df)} ({off_ratio:.02%}) records whose " + "calculations don't match. Adding correction records to make calculations " + "match reported values." + ) + corrections = off_df.copy() + + corrections[self.value_col] = ( + corrections[self.value_col].fillna(0.0) + - corrections["calculated_amount"] + ) + corrections["original_factoid"] = corrections["xbrl_factoid"] + corrections["xbrl_factoid"] = corrections["xbrl_factoid"] + "_correction" + corrections["row_type_xbrl"] = "correction" + corrections["is_within_table_calc"] = False + corrections["record_id"] = pd.NA + + calculated_df = pd.concat( + [calculated_df, corrections], axis="index" + ).reset_index(drop=True) return calculated_df diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 1cd60a7638..42fd3aa2de 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -749,7 +749,6 @@ class ReconcileTableCalculations(TransformParams): def reconcile_table_calculations( df: pd.DataFrame, calculation_components: pd.DataFrame, - xbrl_metadata: pd.DataFrame, xbrl_factoid_name: str, table_name: str, params: ReconcileTableCalculations, @@ -804,9 +803,7 @@ def reconcile_table_calculations( calc_idx = ["xbrl_factoid", "table_name"] + dim_cols if dim_cols: - table_dims = ( - df[calc_idx].drop_duplicates(keep="first").assign(table_name=table_name) - ) + table_dims = df[calc_idx].drop_duplicates(keep="first") # need to add in the correction dimensions. they don't show up in the data at # this point so we don't have the dimensions yet. NOTE: this could have been # done by adding the dims into table_dims..... maybe would have been more @@ -842,9 +839,11 @@ def reconcile_table_calculations( & intra_tbl_calcs[f"{dim}_parent"].isnull() ) ] + pks = pudl.metadata.classes.Resource.from_id(table_name).schema.primary_key calculated_df = calculate_values_from_components( data=df, calculation_components=intra_tbl_calcs, + validate="one_to_many", calc_idx=calc_idx, value_col=params.column_to_check, ) @@ -856,39 +855,83 @@ def reconcile_table_calculations( add_corrections=add_corrections, ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) - # Check that sub-total calculations sum to total. - if params.subtotal_column is not None: + calculated_df = calculated_df.assign( + abs_diff=lambda x: abs(x[params.column_to_check] - x.calculated_amount), + rel_diff=lambda x: np.where( + (x[params.column_to_check] != 0.0), + abs(x.abs_diff / x[params.column_to_check]), + np.nan, + ), + ) + + off_df = calculated_df[ + ~np.isclose( + calculated_df.calculated_amount, calculated_df[params.column_to_check] + ) + & (calculated_df["abs_diff"].notnull()) + ] + calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] + off_ratio = len(off_df) / len(calculated_values) + + if off_ratio > params.calculation_tolerance: + raise AssertionError( + f"Calculations in {table_name} are off by {off_ratio}. Expected tolerance " + f"of {params.calculation_tolerance}." + ) + + # We'll only get here if the proportion of calculations that are off is acceptable + if off_ratio > 0: logger.info( - f"Checking total-to-subtotal calculations within {params.subtotal_column}" + f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " + "calculations don't match. Adding correction records to make calculations " + "match reported values." ) - meta_w_dims = xbrl_metadata.assign( - **{dim: pd.NA for dim in dim_cols} | {"table_name": table_name} - ).pipe( - make_calculation_dimensions_explicit, - table_dimensions_ferc1=table_dims, - dimensions=dim_cols, + corrections = off_df.copy() + corrections[params.column_to_check] = ( + corrections[params.column_to_check].fillna(0.0) + - corrections["calculated_amount"] ) - calc_comps_w_totals = infer_intra_factoid_totals( - intra_tbl_calcs, - meta_w_dims=meta_w_dims, - table_dimensions=table_dims, - dimensions=dim_cols, + corrections[xbrl_factoid_name] = corrections[xbrl_factoid_name] + "_correction" + corrections["row_type_xbrl"] = "correction" + corrections["is_within_table_calc"] = True + corrections["record_id"] = pd.NA + + calculated_df = pd.concat( + [calculated_df, corrections], axis="index" + ).reset_index() + + # Check that sub-total calculations sum to total. + if params.subtotal_column is not None: + sub_group_col = params.subtotal_column + pks_wo_subgroup = [col for col in pks if col != sub_group_col] + calculated_df["sub_total_sum"] = ( + calculated_df.pipe(lambda df: df[df[sub_group_col] != "total"]) + .groupby(pks_wo_subgroup)[params.column_to_check] + .transform("sum") # For each group, calculate sum of sub-components + ) + calculated_df["sub_total_sum"] = calculated_df["sub_total_sum"].fillna( + calculated_df[params.column_to_check] # Fill in value from 'total' column + ) + sub_total_errors = ( + calculated_df.groupby(pks_wo_subgroup) + # If subcomponent sum != total sum, we have nunique()>1 + .filter(lambda x: x["sub_total_sum"].nunique() > 1).groupby( # noqa: PD101 + pks_wo_subgroup + ) + ) + off_ratio_sub = ( + sub_total_errors.ngroups / calculated_df.groupby(pks_wo_subgroup).ngroups ) - subtotal_calcs = calculate_values_from_components( - data=df, - calculation_components=calc_comps_w_totals[ - calc_comps_w_totals.is_total_to_subdimensions_calc - ], - calc_idx=calc_idx, - value_col=params.column_to_check, - ) - subtotal_calcs = check_calculation_metrics( - calculated_df=subtotal_calcs, - value_col=params.column_to_check, - calculation_tolerance=params.calculation_tolerance, - table_name=table_name, - add_corrections=True, - ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) + if sub_total_errors.ngroups > 0: + logger.warning( + f"{table_name}: has {sub_total_errors.ngroups} ({off_ratio_sub:.02%}) sub-total calculations that don't " + "sum to the equivalent total column." + ) + if off_ratio_sub > params.subtotal_calculation_tolerance: + raise AssertionError( + f"Sub-total calculations in {table_name} are off by {off_ratio_sub}. Expected tolerance " + f"of {params.subtotal_calculation_tolerance}." + ) return calculated_df @@ -896,6 +939,7 @@ def reconcile_table_calculations( def calculate_values_from_components( calculation_components: pd.DataFrame, data: pd.DataFrame, + validate: Literal["one_to_many", "many_to_many"], calc_idx: list[str], value_col: str, ) -> pd.DataFrame: @@ -909,6 +953,8 @@ def calculate_values_from_components( data: exploded FERC data to apply the calculations to. Primary key should be ``report_year``, ``utility_id_ferc1``, ``table_name``, ``xbrl_factoid``, and whatever additional dimensions are relevant to the data. + validate: type of merge validation to apply when initially merging the calculation + components (left) and the data (right). calc_idx: primary key columns that uniquely identify a calculation component (not including the ``_parent`` columns). value_col: label of the column in ``data`` that contains the values to apply the @@ -935,7 +981,7 @@ def calculate_values_from_components( pd.merge( calculation_components, data, - validate="one_to_many", + validate=validate, on=calc_idx, ) # apply the weight from the calc to convey the sign before summing. @@ -2393,7 +2439,6 @@ def reconcile_table_calculations( df=df, calculation_components=self.xbrl_calculations, xbrl_factoid_name=self.params.xbrl_factoid_name, - xbrl_metadata=self.xbrl_metadata, table_name=self.table_id.value, params=params, add_corrections=True, @@ -6040,12 +6085,7 @@ def infer_intra_factoid_totals( lambda dim: f"{dim}_parent", axis="columns" ) inferred_totals = inferred_totals.fillna(child_values) - calcs_with_totals = pd.concat( - [ - calc_components.assign(is_total_to_subdimensions_calc=False), - inferred_totals.assign(is_total_to_subdimensions_calc=True), - ] - ) + calcs_with_totals = pd.concat([calc_components, inferred_totals]) # verification + deduping below. diff --git a/test/unit/transform/ferc1_test.py b/test/unit/transform/ferc1_test.py index a942c436c3..b366a4ceb4 100644 --- a/test/unit/transform/ferc1_test.py +++ b/test/unit/transform/ferc1_test.py @@ -451,6 +451,7 @@ def test_calculate_values_from_components(): out_ksr = calculate_values_from_components( calculation_components=calculation_components_ksr, data=data_ksr, + validate="one_to_many", calc_idx=["table_name", "xbrl_factoid", "planet"], value_col="value", ) @@ -662,20 +663,20 @@ def test_adding_parent_dimensions(): pd.read_csv( StringIO( """ -table_name_parent,xbrl_factoid_parent,dim_x_parent,dim_y_parent,table_name,xbrl_factoid,dim_x,dim_y,is_within_table_calc,weight,is_total_to_subdimensions_calc -table_a,fact_1,voyager,coffee,table_a,fact_3,voyager,coffee,True,2,False -table_a,fact_1,voyager,in,table_a,fact_3,voyager,in,True,2,False -table_a,fact_1,voyager,that,table_a,fact_3,voyager,that,True,2,False -table_a,fact_1,voyager,nebula,table_a,fact_3,voyager,nebula,True,2,False -table_a,fact_1,voyager,total,table_a,fact_3,voyager,total,True,2,False -table_a,fact_1,voyager,total,table_a,fact_1,voyager,coffee,True,1,True -table_a,fact_1,voyager,total,table_a,fact_1,voyager,in,True,1,True -table_a,fact_1,voyager,total,table_a,fact_1,voyager,that,True,1,True -table_a,fact_1,voyager,total,table_a,fact_1,voyager,nebula,True,1,True -table_a,fact_3,voyager,total,table_a,fact_3,voyager,coffee,True,1,True -table_a,fact_3,voyager,total,table_a,fact_3,voyager,in,True,1,True -table_a,fact_3,voyager,total,table_a,fact_3,voyager,that,True,1,True -table_a,fact_3,voyager,total,table_a,fact_3,voyager,nebula,True,1,True +table_name_parent,xbrl_factoid_parent,dim_x_parent,dim_y_parent,table_name,xbrl_factoid,dim_x,dim_y,is_within_table_calc,weight +table_a,fact_1,voyager,coffee,table_a,fact_3,voyager,coffee,True,2 +table_a,fact_1,voyager,in,table_a,fact_3,voyager,in,True,2 +table_a,fact_1,voyager,that,table_a,fact_3,voyager,that,True,2 +table_a,fact_1,voyager,nebula,table_a,fact_3,voyager,nebula,True,2 +table_a,fact_1,voyager,total,table_a,fact_3,voyager,total,True,2 +table_a,fact_1,voyager,total,table_a,fact_1,voyager,coffee,True,1 +table_a,fact_1,voyager,total,table_a,fact_1,voyager,in,True,1 +table_a,fact_1,voyager,total,table_a,fact_1,voyager,that,True,1 +table_a,fact_1,voyager,total,table_a,fact_1,voyager,nebula,True,1 +table_a,fact_3,voyager,total,table_a,fact_3,voyager,coffee,True,1 +table_a,fact_3,voyager,total,table_a,fact_3,voyager,in,True,1 +table_a,fact_3,voyager,total,table_a,fact_3,voyager,that,True,1 +table_a,fact_3,voyager,total,table_a,fact_3,voyager,nebula,True,1 """ ) ) @@ -789,19 +790,19 @@ def test_multi_dims_totals(): pd.read_csv( StringIO( """ -table_name_parent,xbrl_factoid_parent,utility_type_parent,plant_status_parent,plant_function_parent,table_name,xbrl_factoid,utility_type,plant_status,plant_function,is_within_table_calc,weight,is_total_to_subdimensions_calc -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1,True -electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1,True +table_name_parent,xbrl_factoid_parent,utility_type_parent,plant_status_parent,plant_function_parent,table_name,xbrl_factoid,utility_type,plant_status,plant_function,is_within_table_calc,weight +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,steam_production,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,total,general,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,steam_production,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,in_service,general,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,steam_production,True,1 +electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,total,electric_plant_depreciation_change_ferc1,accumulated_depreciation,electric,future,general,True,1 """ ) ) From 05b2c4d4e707b364f11e3d0b8e506f521fbf24f1 Mon Sep 17 00:00:00 2001 From: thinky Date: Mon, 25 Sep 2023 17:45:20 -0400 Subject: [PATCH 15/20] Remove total from balance sheet liabilities table --- .../package_data/ferc1/xbrl_calculation_component_fixes.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv index 8d9ee3896a..2f8d626626 100644 --- a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv +++ b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv @@ -31,7 +31,7 @@ balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,accumula balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,,,, balance_sheet_assets_ferc1,utility_plant_net,utility_plant_summary_ferc1,utility_plant_net,1,total,, balance_sheet_liabilities_ferc1,deferred_credits,balance_sheet_liabilities_ferc1,accumulated_deferred_income_taxes,1,,, -balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,1,total,, +balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,1,,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities,,,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities,-1,,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities_hedges,,,, From c25ed15217e7406d4ffd7c7e3302fbeac05384bb Mon Sep 17 00:00:00 2001 From: thinky Date: Mon, 25 Sep 2023 18:07:03 -0400 Subject: [PATCH 16/20] fix rebase --- src/pudl/output/ferc1.py | 156 +++--------------------------------- src/pudl/transform/ferc1.py | 45 ----------- 2 files changed, 13 insertions(+), 188 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 54801ec5a1..0ceaac365c 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1369,10 +1369,6 @@ def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame exploded = ( self.initial_explosion_concatenation(tables_to_explode) .pipe(self.generate_intertable_calculations) - .pipe( - self.reconcile_intertable_calculations, - self.calculation_tolerance.intertable_calculation_errors, - ) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... @@ -1469,147 +1465,21 @@ def generate_intertable_calculations( f"{self.root_table}: Reconcile inter-table calculations: " f"{list(calculations_intertable.xbrl_factoid.unique())}." ) - # compile the lists of columns we are going to use later - calc_component_idx = ["table_name", "xbrl_factoid"] + self.other_dimensions - # Merge the reported data and the calculation component metadata to enable - # validation of calculated values. Here the data table exploded is supplying the - # values associated with individual calculation components, and the table_name - # and xbrl_factoid to which we aggregate are coming from the calculation - # components table. After merging we use the weights to adjust the reported - # values so they can be summed directly. This gives us aggregated calculated - # values that can later be compared to the higher level reported values. - - # the validation is one_many in all instances expect for the xbrl_factoid - # construction_work_in_progress in the balance_sheet_assets_ferc1 explosion. - # this may be a problem in the calculations that we should track down in #2717 - validate = ( - "one_to_many" - if self.root_table != "balance_sheet_assets_ferc1" - else "many_to_many" - ) - # we are going to merge the data onto the calc components with the _parent - # column names, so the groupby after the merge needs a set of by cols with the - # _parent suffix - meta_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] - gby_parent = [ - f"{col}_parent" if col in meta_idx else col for col in self.exploded_pks - ] - calc_df = ( - pd.merge( - calculations_intertable, - exploded, - validate=validate, - on=calc_component_idx, - ) - # apply the weight from the calc to convey the sign before summing. - .assign(calculated_amount=lambda x: x[self.value_col] * x.weight) - .groupby(gby_parent, as_index=False, dropna=False)[["calculated_amount"]] - .sum(min_count=1) - ) - # remove the _parent suffix so we can merge these calculated values back onto - # the data using the original pks - calc_df.columns = calc_df.columns.str.removesuffix("_parent") - calculated_df = pd.merge( - exploded, - calc_df, - on=self.exploded_pks, - how="outer", - validate="1:1", - indicator=True, - ) - - assert calculated_df[ - (calculated_df._merge == "right_only") - & (calculated_df[self.value_col].notnull()) - ].empty - - calculated_df = calculated_df.drop(columns=["_merge"]) - # Force value_col to be a float to prevent any hijinks with calculating differences. - calculated_df[self.value_col] = calculated_df[self.value_col].astype(float) - - return calculated_df - - def reconcile_intertable_calculations( - self: Self, calculated_df: pd.DataFrame, calculation_tolerance: float = 0.05 - ): - """Ensure inter-table calculated values match reported values within a tolerance. - - In addition to checking whether all reported "calculated" values match the output - of our repaired calculations, this function adds a correction record to the - dataframe that is included in the calculations so that after the fact the - calculations match exactly. This is only done when the fraction of records that - don't match within the tolerances of :func:`numpy.isclose` is below a set - threshold. - - Note that only calculations which are off by a significant amount result in the - creation of a correction record. Many calculations are off from the reported values - by exaclty one dollar, presumably due to rounding errrors. These records typically - do not fail the :func:`numpy.isclose()` test and so are not corrected. - - Args: - calculated_df: table with calculated fields - calculation_tolerance: What proportion (0-1) of calculated values are - allowed to be incorrect without raising an AssertionError. - """ - if "calculated_amount" not in calculated_df.columns: - return calculated_df - - # Data types were very messy here, including pandas Float64 for the - # calculated_amount columns which did not work with the np.isclose(). Not sure - # why these are cropping up. - calculated_df = calculated_df.convert_dtypes(convert_floating=False).astype( - {self.value_col: "float64", "calculated_amount": "float64"} + calc_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] + calculated_df = pudl.transform.ferc1.calculate_values_from_components( + calculation_components=calculations_intertable, + data=exploded, + validate="one_to_many", + calc_idx=calc_idx, + value_col=self.value_col, ) - calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[self.value_col] - x.calculated_amount), - rel_diff=lambda x: np.where( - (x[self.value_col] != 0.0), - abs(x.abs_diff / x[self.value_col]), - np.nan, - ), + calculated_df = pudl.transform.ferc1.check_calculation_metrics( + calculated_df=calculated_df, + value_col=self.value_col, + calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, + table_name=self.root_table, + add_corrections=True, ) - off_df = calculated_df[ - ~np.isclose(calculated_df.calculated_amount, calculated_df[self.value_col]) - & (calculated_df["abs_diff"].notnull()) - ] - calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] - if calculated_values.empty: - # Will only occur if all reported values are NaN when calculated values - # exist, or vice versa. - logger.warning( - "Warning: No calculated values have a corresponding reported value in the table." - ) - off_ratio = np.nan - else: - off_ratio = len(off_df) / len(calculated_values) - if off_ratio > calculation_tolerance: - raise AssertionError( - f"Calculations in {self.root_table} are off by {off_ratio:.2%}. Expected tolerance " - f"of {calculation_tolerance:.1%}." - ) - - # # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0 or np.isnan(off_ratio): - logger.info( - f"{self.root_table}: has {len(off_df)} ({off_ratio:.02%}) records whose " - "calculations don't match. Adding correction records to make calculations " - "match reported values." - ) - corrections = off_df.copy() - - corrections[self.value_col] = ( - corrections[self.value_col].fillna(0.0) - - corrections["calculated_amount"] - ) - corrections["original_factoid"] = corrections["xbrl_factoid"] - corrections["xbrl_factoid"] = corrections["xbrl_factoid"] + "_correction" - corrections["row_type_xbrl"] = "correction" - corrections["is_within_table_calc"] = False - corrections["record_id"] = pd.NA - - calculated_df = pd.concat( - [calculated_df, corrections], axis="index" - ).reset_index(drop=True) return calculated_df diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 42fd3aa2de..2bfae0ff5b 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -855,51 +855,6 @@ def reconcile_table_calculations( add_corrections=add_corrections, ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) - calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[params.column_to_check] - x.calculated_amount), - rel_diff=lambda x: np.where( - (x[params.column_to_check] != 0.0), - abs(x.abs_diff / x[params.column_to_check]), - np.nan, - ), - ) - - off_df = calculated_df[ - ~np.isclose( - calculated_df.calculated_amount, calculated_df[params.column_to_check] - ) - & (calculated_df["abs_diff"].notnull()) - ] - calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] - off_ratio = len(off_df) / len(calculated_values) - - if off_ratio > params.calculation_tolerance: - raise AssertionError( - f"Calculations in {table_name} are off by {off_ratio}. Expected tolerance " - f"of {params.calculation_tolerance}." - ) - - # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0: - logger.info( - f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " - "calculations don't match. Adding correction records to make calculations " - "match reported values." - ) - corrections = off_df.copy() - corrections[params.column_to_check] = ( - corrections[params.column_to_check].fillna(0.0) - - corrections["calculated_amount"] - ) - corrections[xbrl_factoid_name] = corrections[xbrl_factoid_name] + "_correction" - corrections["row_type_xbrl"] = "correction" - corrections["is_within_table_calc"] = True - corrections["record_id"] = pd.NA - - calculated_df = pd.concat( - [calculated_df, corrections], axis="index" - ).reset_index() - # Check that sub-total calculations sum to total. if params.subtotal_column is not None: sub_group_col = params.subtotal_column From 80035d11db1ede31cafacda2466945da19550bc9 Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 25 Sep 2023 16:08:12 -0600 Subject: [PATCH 17/20] Validate one_to_many in reconcile_calculations; set balance_sheet_assets expected error to 65% --- src/pudl/output/ferc1.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 0ceaac365c..51d0c0bfb0 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -44,7 +44,7 @@ class CalculationToleranceFerc1(BaseModel): intertable_calculation_errors=0.20, ), "balance_sheet_assets_ferc1": CalculationToleranceFerc1( - intertable_calculation_errors=0.85, + intertable_calculation_errors=0.65, ), "balance_sheet_liabilities_ferc1": CalculationToleranceFerc1( intertable_calculation_errors=0.07, @@ -1469,7 +1469,6 @@ def generate_intertable_calculations( calculated_df = pudl.transform.ferc1.calculate_values_from_components( calculation_components=calculations_intertable, data=exploded, - validate="one_to_many", calc_idx=calc_idx, value_col=self.value_col, ) From 33def8fbaf17dd9a5b93f40b44e1c87fe6ca464e Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 25 Sep 2023 16:13:42 -0600 Subject: [PATCH 18/20] Revert to floats in weights to make diffs more obvious. --- .../xbrl_calculation_component_fixes.csv | 180 +++++++++--------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv index 2f8d626626..b0e6b7398f 100644 --- a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv +++ b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv @@ -1,42 +1,42 @@ table_name_parent,xbrl_factoid_parent,table_name,xbrl_factoid,weight,utility_type,plant_function,plant_status balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,abandonment_of_leases,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1,total,, +balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1.0,total,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_other_utility_plant_utility_plant_in_service,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_plant_acquisition_adjustment,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_leased_to_others,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_and_amortization_utility_plant_held_for_future_use,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_utility_plant_in_service,,,, -balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,1,total,, +balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,1.0,total,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_hedges_long_term,,,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_long_term,,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_hedges_long_term,-1,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_long_term,-1,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_noncurrent_portion_of_allowances,-1,,, +balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_hedges_long_term,-1.0,,, +balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_long_term,-1.0,,, +balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_noncurrent_portion_of_allowances,-1.0,,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,noncurrent_portion_of_allowances,,,, -balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_survey_and_investigation_charges,1,,, -balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_and_other_survey_and_investigation_charges,1,,, -balance_sheet_assets_ferc1,nuclear_fuel_net,balance_sheet_assets_ferc1,nuclear_fuel,1,,, +balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_survey_and_investigation_charges,1.0,,, +balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_and_other_survey_and_investigation_charges,1.0,,, +balance_sheet_assets_ferc1,nuclear_fuel_net,balance_sheet_assets_ferc1,nuclear_fuel,1.0,,, balance_sheet_assets_ferc1,nuclear_fuel_net,nuclear_fuel_materials_ferc1,nuclear_fuel_materials_and_assemblies,,,, balance_sheet_assets_ferc1,nuclear_fuel_net,nuclear_fuel_materials_ferc1,spent_nuclear_fuel,,,, -balance_sheet_assets_ferc1,other_property_and_investments,balance_sheet_assets_ferc1,special_funds_all,1,,, +balance_sheet_assets_ferc1,other_property_and_investments,balance_sheet_assets_ferc1,special_funds_all,1.0,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,construction_work_in_progress,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,utility_plant,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_acquisition_adjustment,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,1,total,, +balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,1.0,total,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_held_for_future_use,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,,,, balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_leased_to_others,,,, balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,,,, balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,,,, -balance_sheet_assets_ferc1,utility_plant_net,utility_plant_summary_ferc1,utility_plant_net,1,total,, -balance_sheet_liabilities_ferc1,deferred_credits,balance_sheet_liabilities_ferc1,accumulated_deferred_income_taxes,1,,, -balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,1,,, +balance_sheet_assets_ferc1,utility_plant_net,utility_plant_summary_ferc1,utility_plant_net,1.0,total,, +balance_sheet_liabilities_ferc1,deferred_credits,balance_sheet_liabilities_ferc1,accumulated_deferred_income_taxes,1.0,,, +balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,1.0,,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities,,,, -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities,-1,,, +balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities,-1.0,,, balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities_hedges,,,, -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities_hedges,-1,,, -electric_energy_sources_ferc1,sources_of_energy,electric_energy_sources_ferc1,megawatt_hours_purchased,1,,, +balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities_hedges,-1.0,,, +electric_energy_sources_ferc1,sources_of_energy,electric_energy_sources_ferc1,megawatt_hours_purchased,1.0,,, electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,electric_expenses_hydraulic_power_generation,,,, electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,hydraulic_expenses,,,, electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_of_electric_plant_hydraulic_power_generation,,,, @@ -67,82 +67,82 @@ electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_hydro_ferc1,opex_operations,,,, electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_pumped_storage_ferc1,opex_operations,,,, electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_steam_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,transmission_operation_expense,electric_operating_expenses_ferc1,load_dispatching_transmission_expense,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,forfeited_discounts,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,interdepartmental_rents,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_revenue,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_service_revenues,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_electric_revenue,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_miscellaneous_operating_revenues,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,regional_transmission_service_revenues,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,rent_from_electric_property,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,revenues_from_transmission_of_electricity_of_others,1,,, -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,sales_of_water_and_water_power,1,,, -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,large_or_industrial,1,,, -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,small_or_commercial,1,,, +electric_operating_expenses_ferc1,transmission_operation_expense,electric_operating_expenses_ferc1,load_dispatching_transmission_expense,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,forfeited_discounts,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,interdepartmental_rents,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_revenue,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_service_revenues,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_electric_revenue,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_miscellaneous_operating_revenues,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,regional_transmission_service_revenues,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,rent_from_electric_property,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,revenues_from_transmission_of_electricity_of_others,1.0,,, +electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,sales_of_water_and_water_power,1.0,,, +electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,large_or_industrial,1.0,,, +electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,small_or_commercial,1.0,,, electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electricity_sales_by_rate_schedule_ferc1,commercial_and_industrial,,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,book_cost_of_asset_retirement_costs,1,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,depreciation_provision,1,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,net_charges_for_retired_plant,1,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,other_adjustments_to_accumulated_depreciation,1,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,starting_balance,1,,, -income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_limited_term_electric_plant,1,electric,total, -income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_other_electric_plant,1,electric,total, -income_statement_ferc1,depreciation_expense,depreciation_amortization_summary_ferc1,depreciation_expense,1,electric,total, -income_statement_ferc1,depreciation_expense_for_asset_retirement_costs,depreciation_amortization_summary_ferc1,depreciation_expense_asset_retirement,1,electric,total, -income_statement_ferc1,income_before_extraordinary_items,income_statement_ferc1,net_utility_operating_income,1,,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,distribution_maintenance_expense_electric,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_maintenance_expense,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,maintenance_of_general_plant,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,nuclear_power_generation_maintenance_expense,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,other_power_generation_maintenance_expense,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,regional_market_maintenance_expense,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,steam_power_generation_maintenance_expense,1,electric,, -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,transmission_maintenance_expense_electric,1,electric,, -income_statement_ferc1,operating_revenues,electric_operating_revenues_ferc1,electric_operating_revenues,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,administrative_and_general_operation_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_account_expenses,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_service_and_information_expenses,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,distribution_operation_expenses_electric,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_operations_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,nuclear_power_generation_operations_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,other_power_generation_operations_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,regional_market_operation_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,sales_expenses,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,steam_power_generation_operations_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,transmission_operation_expense,1,electric,, -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,power_production_expenses,1,electric,, -income_statement_ferc1,other_income_deductions,income_statement_ferc1,miscellaneous_deductions,1,,, -income_statement_ferc1,taxes_on_other_income_and_deductions,income_statement_ferc1,investment_tax_credits,-1,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,distribution_plant,1,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,general_plant,1,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,intangible_plant,1,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,production_plant,1,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,1,,, -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_plant,1,,, -retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings,1,,, -retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings_amortization_reserve_federal,1,,, -retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,1,,, -retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_credit,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_debit,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,appropriations_of_retained_earnings,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,balance_transferred_from_income,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_common_stock,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_preferred_stock,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,transfers_from_unappropriated_undistributed_subsidiary_earnings,1,,, -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings_previous_year,1,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,changes_unappropriated_undistributed_subsidiary_earnings_credits,1,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,dividends_received,-1,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,equity_in_earnings_of_subsidiary_companies,1,,, -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings_previous_year,1,,, -balance_sheet_liabilities_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,1,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,book_cost_of_asset_retirement_costs,1.0,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,depreciation_provision,1.0,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,net_charges_for_retired_plant,1.0,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,other_adjustments_to_accumulated_depreciation,1.0,,, +electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,starting_balance,1.0,,, +income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_limited_term_electric_plant,1.0,electric,total, +income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_other_electric_plant,1.0,electric,total, +income_statement_ferc1,depreciation_expense,depreciation_amortization_summary_ferc1,depreciation_expense,1.0,electric,total, +income_statement_ferc1,depreciation_expense_for_asset_retirement_costs,depreciation_amortization_summary_ferc1,depreciation_expense_asset_retirement,1.0,electric,total, +income_statement_ferc1,income_before_extraordinary_items,income_statement_ferc1,net_utility_operating_income,1.0,,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,distribution_maintenance_expense_electric,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_maintenance_expense,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,maintenance_of_general_plant,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,nuclear_power_generation_maintenance_expense,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,other_power_generation_maintenance_expense,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,regional_market_maintenance_expense,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,steam_power_generation_maintenance_expense,1.0,electric,, +income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,transmission_maintenance_expense_electric,1.0,electric,, +income_statement_ferc1,operating_revenues,electric_operating_revenues_ferc1,electric_operating_revenues,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,administrative_and_general_operation_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_account_expenses,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_service_and_information_expenses,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,distribution_operation_expenses_electric,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_operations_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,nuclear_power_generation_operations_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,other_power_generation_operations_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,regional_market_operation_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,sales_expenses,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,steam_power_generation_operations_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,transmission_operation_expense,1.0,electric,, +income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,power_production_expenses,1.0,electric,, +income_statement_ferc1,other_income_deductions,income_statement_ferc1,miscellaneous_deductions,1.0,,, +income_statement_ferc1,taxes_on_other_income_and_deductions,income_statement_ferc1,investment_tax_credits,-1.0,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,distribution_plant,1.0,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,general_plant,1.0,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,intangible_plant,1.0,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,production_plant,1.0,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,1.0,,, +plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_plant,1.0,,, +retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings,1.0,,, +retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings_amortization_reserve_federal,1.0,,, +retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,1.0,,, +retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_credit,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_debit,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,appropriations_of_retained_earnings,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,balance_transferred_from_income,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_common_stock,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_preferred_stock,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,transfers_from_unappropriated_undistributed_subsidiary_earnings,1.0,,, +retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings_previous_year,1.0,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,changes_unappropriated_undistributed_subsidiary_earnings_credits,1.0,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,dividends_received,-1.0,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,equity_in_earnings_of_subsidiary_companies,1.0,,, +retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings_previous_year,1.0,,, +balance_sheet_liabilities_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,1.0,,, utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, -utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,1,electric,total,in_service +utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,1.0,electric,total,in_service utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified,,,, -utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_property_under_capital_leases,1,,, +utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_property_under_capital_leases,1.0,,, utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_property_under_capital_leases,,,, -utility_plant_summary_ferc1,utility_plant_in_service_experimental_plant_unclassified,plant_in_service_ferc1,experimental_electric_plant_unclassified,1,electric,, -utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_purchased,1,electric,, -utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_sold,-1,electric,, +utility_plant_summary_ferc1,utility_plant_in_service_experimental_plant_unclassified,plant_in_service_ferc1,experimental_electric_plant_unclassified,1.0,electric,, +utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_purchased,1.0,electric,, +utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_sold,-1.0,electric,, utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,utility_plant,,,, From 0c1ff76cecd82186147557a2573cf08324189dc7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 05:17:14 +0000 Subject: [PATCH 19/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.290 → v0.0.291](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.290...v0.0.291) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18eecbaf2d..147e578003 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: # Formatters: hooks that re-write Python & documentation files #################################################################################### - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.290 + rev: v0.0.291 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From f3364b60bf1c61b0f4a022a98c66b941bc0a37f8 Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Tue, 26 Sep 2023 09:20:52 -0600 Subject: [PATCH 20/20] Pin to astroid<3 to avoid sphinx-autoapi compatibility issues. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 738375e10c..7ab1e511c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ license = { file = "LICENSE.txt" } dependencies = [ "addfips>=0.4,<0.5", "alembic>=1.10.3,<1.13", + "astroid>=2,<3", "catalystcoop.dbfread>=3.0,<3.1", "catalystcoop.ferc-xbrl-extractor==0.8.3", "coloredlogs>=14.0,<15.1", # Dagster requires 14.0