diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18eecbaf2d..147e578003 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: # Formatters: hooks that re-write Python & documentation files #################################################################################### - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.290 + rev: v0.0.291 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/pyproject.toml b/pyproject.toml index a96fceea01..7ab1e511c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,15 @@ license = { file = "LICENSE.txt" } dependencies = [ "addfips>=0.4,<0.5", "alembic>=1.10.3,<1.13", + "astroid>=2,<3", "catalystcoop.dbfread>=3.0,<3.1", "catalystcoop.ferc-xbrl-extractor==0.8.3", "coloredlogs>=14.0,<15.1", # Dagster requires 14.0 "dagster-webserver>=1.4,<1.5", # 1.2.2 is first version to support Python 3.11 "dagster>=1.4,<1.5", # 1.2.2 is first version to support Python 3.11 - "dask>=2021.8,<2023.9.2", + "dask>=2021.8,<2023.9.3", "datapackage>=1.11,<1.16", # Transition datastore to use frictionless. - "fsspec>=2021.7,<2023.6.1", # For caching datastore on GCS + "fsspec>=2021.7,<2023.9.3", # For caching datastore on GCS "geopandas>=0.13,<0.15", "gcsfs>=2021.7,<2023.9.2", # For caching datastore on GCS "grpcio<1.56.0", diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 9ff916d221..51d0c0bfb0 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -44,7 +44,7 @@ class CalculationToleranceFerc1(BaseModel): intertable_calculation_errors=0.20, ), "balance_sheet_assets_ferc1": CalculationToleranceFerc1( - intertable_calculation_errors=0.85, + intertable_calculation_errors=0.65, ), "balance_sheet_liabilities_ferc1": CalculationToleranceFerc1( intertable_calculation_errors=0.07, @@ -1369,10 +1369,6 @@ def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame exploded = ( self.initial_explosion_concatenation(tables_to_explode) .pipe(self.generate_intertable_calculations) - .pipe( - self.reconcile_intertable_calculations, - self.calculation_tolerance.intertable_calculation_errors, - ) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... @@ -1469,147 +1465,20 @@ def generate_intertable_calculations( f"{self.root_table}: Reconcile inter-table calculations: " f"{list(calculations_intertable.xbrl_factoid.unique())}." ) - # compile the lists of columns we are going to use later - calc_component_idx = ["table_name", "xbrl_factoid"] + self.other_dimensions - # Merge the reported data and the calculation component metadata to enable - # validation of calculated values. Here the data table exploded is supplying the - # values associated with individual calculation components, and the table_name - # and xbrl_factoid to which we aggregate are coming from the calculation - # components table. After merging we use the weights to adjust the reported - # values so they can be summed directly. This gives us aggregated calculated - # values that can later be compared to the higher level reported values. - - # the validation is one_many in all instances expect for the xbrl_factoid - # construction_work_in_progress in the balance_sheet_assets_ferc1 explosion. - # this may be a problem in the calculations that we should track down in #2717 - validate = ( - "one_to_many" - if self.root_table != "balance_sheet_assets_ferc1" - else "many_to_many" - ) - # we are going to merge the data onto the calc components with the _parent - # column names, so the groupby after the merge needs a set of by cols with the - # _parent suffix - meta_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] - gby_parent = [ - f"{col}_parent" if col in meta_idx else col for col in self.exploded_pks - ] - calc_df = ( - pd.merge( - calculations_intertable, - exploded, - validate=validate, - on=calc_component_idx, - ) - # apply the weight from the calc to convey the sign before summing. - .assign(calculated_amount=lambda x: x[self.value_col] * x.weight) - .groupby(gby_parent, as_index=False, dropna=False)[["calculated_amount"]] - .sum(min_count=1) - ) - # remove the _parent suffix so we can merge these calculated values back onto - # the data using the original pks - calc_df.columns = calc_df.columns.str.removesuffix("_parent") - calculated_df = pd.merge( - exploded, - calc_df, - on=self.exploded_pks, - how="outer", - validate="1:1", - indicator=True, - ) - - assert calculated_df[ - (calculated_df._merge == "right_only") - & (calculated_df[self.value_col].notnull()) - ].empty - - calculated_df = calculated_df.drop(columns=["_merge"]) - # Force value_col to be a float to prevent any hijinks with calculating differences. - calculated_df[self.value_col] = calculated_df[self.value_col].astype(float) - - return calculated_df - - def reconcile_intertable_calculations( - self: Self, calculated_df: pd.DataFrame, calculation_tolerance: float = 0.05 - ): - """Ensure inter-table calculated values match reported values within a tolerance. - - In addition to checking whether all reported "calculated" values match the output - of our repaired calculations, this function adds a correction record to the - dataframe that is included in the calculations so that after the fact the - calculations match exactly. This is only done when the fraction of records that - don't match within the tolerances of :func:`numpy.isclose` is below a set - threshold. - - Note that only calculations which are off by a significant amount result in the - creation of a correction record. Many calculations are off from the reported values - by exaclty one dollar, presumably due to rounding errrors. These records typically - do not fail the :func:`numpy.isclose()` test and so are not corrected. - - Args: - calculated_df: table with calculated fields - calculation_tolerance: What proportion (0-1) of calculated values are - allowed to be incorrect without raising an AssertionError. - """ - if "calculated_amount" not in calculated_df.columns: - return calculated_df - - # Data types were very messy here, including pandas Float64 for the - # calculated_amount columns which did not work with the np.isclose(). Not sure - # why these are cropping up. - calculated_df = calculated_df.convert_dtypes(convert_floating=False).astype( - {self.value_col: "float64", "calculated_amount": "float64"} + calc_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] + calculated_df = pudl.transform.ferc1.calculate_values_from_components( + calculation_components=calculations_intertable, + data=exploded, + calc_idx=calc_idx, + value_col=self.value_col, ) - calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[self.value_col] - x.calculated_amount), - rel_diff=lambda x: np.where( - (x[self.value_col] != 0.0), - abs(x.abs_diff / x[self.value_col]), - np.nan, - ), + calculated_df = pudl.transform.ferc1.check_calculation_metrics( + calculated_df=calculated_df, + value_col=self.value_col, + calculation_tolerance=self.calculation_tolerance.intertable_calculation_errors, + table_name=self.root_table, + add_corrections=True, ) - off_df = calculated_df[ - ~np.isclose(calculated_df.calculated_amount, calculated_df[self.value_col]) - & (calculated_df["abs_diff"].notnull()) - ] - calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] - if calculated_values.empty: - # Will only occur if all reported values are NaN when calculated values - # exist, or vice versa. - logger.warning( - "Warning: No calculated values have a corresponding reported value in the table." - ) - off_ratio = np.nan - else: - off_ratio = len(off_df) / len(calculated_values) - if off_ratio > calculation_tolerance: - raise AssertionError( - f"Calculations in {self.root_table} are off by {off_ratio:.2%}. Expected tolerance " - f"of {calculation_tolerance:.1%}." - ) - - # # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0 or np.isnan(off_ratio): - logger.info( - f"{self.root_table}: has {len(off_df)} ({off_ratio:.02%}) records whose " - "calculations don't match. Adding correction records to make calculations " - "match reported values." - ) - corrections = off_df.copy() - - corrections[self.value_col] = ( - corrections[self.value_col].fillna(0.0) - - corrections["calculated_amount"] - ) - corrections["original_factoid"] = corrections["xbrl_factoid"] - corrections["xbrl_factoid"] = corrections["xbrl_factoid"] + "_correction" - corrections["row_type_xbrl"] = "correction" - corrections["is_within_table_calc"] = False - corrections["record_id"] = pd.NA - - calculated_df = pd.concat( - [calculated_df, corrections], axis="index" - ).reset_index(drop=True) return calculated_df @@ -1688,21 +1557,6 @@ def unique_associations(cls, v: pd.DataFrame, values) -> pd.DataFrame: assert not v.duplicated(subset=pks, keep=False).any() return v - @validator("exploded_calcs") - def single_valued_weights(cls, v: pd.DataFrame, values) -> pd.DataFrame: - """Ensure that every calculation component has a uniquely specified weight.""" - multi_valued_weights = ( - v.groupby(values["calc_cols"], dropna=False)["weight"] - .transform("nunique") - .gt(1) - ) - if multi_valued_weights.any(): - logger.warning( - f"Found {sum(multi_valued_weights)} calculations with conflicting " - "weights." - ) - return v - @validator("exploded_calcs") def calcs_have_required_cols(cls, v: pd.DataFrame, values) -> pd.DataFrame: """Ensure exploded calculations include all required columns.""" diff --git a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv index b68db70768..b0e6b7398f 100644 --- a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv +++ b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv @@ -1,13 +1,13 @@ table_name_parent,xbrl_factoid_parent,table_name,xbrl_factoid,weight,utility_type,plant_function,plant_status balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,abandonment_of_leases,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1.0,,, +balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1.0,total,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_other_utility_plant_utility_plant_in_service,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_plant_acquisition_adjustment,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_leased_to_others,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_and_amortization_utility_plant_held_for_future_use,,,, balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_utility_plant_in_service,,,, -balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,1.0,,, +balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,1.0,total,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_hedges_long_term,,,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_long_term,,,, balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_hedges_long_term,-1.0,,, @@ -138,7 +138,7 @@ retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retaine retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings_previous_year,1.0,,, balance_sheet_liabilities_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,1.0,,, utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, -utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,1.0,electric,total,total +utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,1.0,electric,total,in_service utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified,,,, utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_property_under_capital_leases,1.0,,, utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_property_under_capital_leases,,,, diff --git a/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv b/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv index 0ea8932c91..705ec73e2f 100644 --- a/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv +++ b/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv @@ -1,7 +1,7 @@ xbrl_factoid_rmi,xbrl_factoid_rmi_renamed,xbrl_factoid,table_name,in_rate_base,utility_type,plant_function,plant_status,notes accrued_utility_revenues,accrued_utility_revenues,accrued_utility_revenues,balance_sheet_assets_ferc1,yes,,,,"At the option of the utility, the estimated amount accrued for service rendered, but not billed at the end of any accounting period, may be included herein. In case accruals are made for unbilled revenues, they shall be made likewise for unbilled expenses, such as for the purchase of energy." advances_for_gas,advances_for_gas,advances_for_gas,balance_sheet_assets_ferc1,yes,,,,"4/13/21 - changed from ""non-electric"" to ""mixed"" - could be for gas power plants." -allowances,allowances,allowances,electric_operating_expenses_ferc1,yes,,,, +allowances,allowances,allowance_inventory_and_withheld,balance_sheet_assets_ferc1,yes,,,, cash,cash,cash,balance_sheet_assets_ferc1,yes,,,, customer_accounts_receivable,customer_accounts_receivable,customer_accounts_receivable,balance_sheet_assets_ferc1,yes,,,,"service provided & billed/collected, asset earned on." derivative_instrument_assets,derivative_instrument_assets,derivative_instrument_assets,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? @@ -115,10 +115,10 @@ unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_su unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,no,,,, accounts_receivable_from_assoc_companies,accounts_receivable_from_assoc_companies,notes_receivable_from_associated_companies,balance_sheet_assets_ferc1,no,,,,Don't believe associated companies are involved in utility services? accum_prov_for_uncollectible_acct_credit,accum_prov_for_uncollectible_acct_credit,accumulated_provision_for_uncollectible_accounts_credit,,yes,,,,Losses on accounts receivable which may become collectible. -cash_and_working_funds_(non_major_only),cash_and_working_funds_(non_major_only),cash_and_working_funds,,yes,,,, -derivative_instrument_assets___hedges,derivative_instrument_assets___hedges,derivative_instrument_assets_hedges,,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +cash_and_working_funds_(non_major_only),cash_and_working_funds_(non_major_only),cash_and_working_funds,balance_sheet_assets_ferc1,yes,,,, +derivative_instrument_assets___hedges,derivative_instrument_assets___hedges,derivative_instrument_assets_hedges,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? gas_stored_underground___current,gas_stored_underground___current,gas_stored_current,balance_sheet_assets_ferc1,yes,,,, -long_term_portion_of_derivative_instrument_assets,long_term_portion_of_derivative_instrument_assets,derivative_instrument_assets_hedges_long_term,,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +long_term_portion_of_derivative_instrument_assets,long_term_portion_of_derivative_instrument_assets,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? long_term_portion_of_derivative_instrument_assets___hedges,long_term_portion_of_derivative_instrument_assets___hedges,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? residuals_(elec)_and_extracted_products,residuals_(elec)_and_extracted_products,residuals,balance_sheet_assets_ferc1,yes,,,, working_fund,working_fund,working_funds,balance_sheet_assets_ferc1,yes,,,, diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 8b29c4bcbe..a0dde28010 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -752,6 +752,7 @@ def reconcile_table_calculations( xbrl_factoid_name: str, table_name: str, params: ReconcileTableCalculations, + add_corrections: bool = True, ) -> pd.DataFrame: """Ensure intra-table calculated values match reported values within a tolerance. @@ -773,6 +774,8 @@ def reconcile_table_calculations( xbrl_factoid_name: column name of the XBRL factoid in the processed table. table_name: name of the PUDL table. params: :class:`ReconcileTableCalculations` parameters. + add_corrections: Whether or not to create _correction records that force all + calculations to add up correctly. """ # If we don't have this value, we aren't doing any calculation checking: if params.column_to_check is None or calculation_components.empty: @@ -840,55 +843,16 @@ def reconcile_table_calculations( calculated_df = calculate_values_from_components( data=df, calculation_components=intra_tbl_calcs, - validate="one_to_many", calc_idx=calc_idx, value_col=params.column_to_check, - ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) - - calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[params.column_to_check] - x.calculated_amount), - rel_diff=lambda x: np.where( - (x[params.column_to_check] != 0.0), - abs(x.abs_diff / x[params.column_to_check]), - np.nan, - ), ) - - off_df = calculated_df[ - ~np.isclose( - calculated_df.calculated_amount, calculated_df[params.column_to_check] - ) - & (calculated_df["abs_diff"].notnull()) - ] - calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] - off_ratio = len(off_df) / len(calculated_values) - - if off_ratio > params.calculation_tolerance: - raise AssertionError( - f"Calculations in {table_name} are off by {off_ratio}. Expected tolerance " - f"of {params.calculation_tolerance}." - ) - - # We'll only get here if the proportion of calculations that are off is acceptable - if off_ratio > 0: - logger.info( - f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " - "calculations don't match. Adding correction records to make calculations " - "match reported values." - ) - corrections = off_df.copy() - corrections[params.column_to_check] = ( - corrections[params.column_to_check].fillna(0.0) - - corrections["calculated_amount"] - ) - corrections[xbrl_factoid_name] = corrections[xbrl_factoid_name] + "_correction" - corrections["row_type_xbrl"] = "correction" - corrections["is_within_table_calc"] = True - corrections["record_id"] = pd.NA - - calculated_df = pd.concat( - [calculated_df, corrections], axis="index" - ).reset_index() + calculated_df = check_calculation_metrics( + calculated_df=calculated_df, + value_col=params.column_to_check, + calculation_tolerance=params.calculation_tolerance, + table_name=table_name, + add_corrections=add_corrections, + ).rename(columns={"xbrl_factoid": xbrl_factoid_name}) # Check that sub-total calculations sum to total. if params.subtotal_column is not None: @@ -929,7 +893,6 @@ def reconcile_table_calculations( def calculate_values_from_components( calculation_components: pd.DataFrame, data: pd.DataFrame, - validate: Literal["one_to_many", "many_to_many"], calc_idx: list[str], value_col: str, ) -> pd.DataFrame: @@ -971,7 +934,7 @@ def calculate_values_from_components( pd.merge( calculation_components, data, - validate=validate, + validate="one_to_many", on=calc_idx, ) # apply the weight from the calc to convey the sign before summing. @@ -1001,6 +964,72 @@ def calculate_values_from_components( return calculated_df +def check_calculation_metrics( + calculated_df: pd.DataFrame, + value_col: str, + calculation_tolerance: float, + table_name: str, + add_corrections: bool = True, +) -> pd.DataFrame: + """Run the calculation metrics and determine if calculations are within tolerance.""" + # Data types were very messy here, including pandas Float64 for the + # calculated_amount columns which did not work with the np.isclose(). Not sure + # why these are cropping up. + calculated_df = calculated_df.convert_dtypes(convert_floating=False).astype( + {value_col: "float64", "calculated_amount": "float64"} + ) + calculated_df = calculated_df.assign( + abs_diff=lambda x: abs(x[value_col] - x.calculated_amount), + rel_diff=lambda x: np.where( + (x[value_col] != 0.0), + abs(x.abs_diff / x[value_col]), + np.nan, + ), + ) + + off_df = calculated_df[ + ~np.isclose(calculated_df.calculated_amount, calculated_df[value_col]) + & (calculated_df["abs_diff"].notnull()) + ] + calculated_values = calculated_df[(calculated_df.abs_diff.notnull())] + if calculated_values.empty: + # Will only occur if all reported values are NaN when calculated values + # exist, or vice versa. + logger.warning( + "Warning: No calculated values have a corresponding reported value in the table." + ) + off_ratio = np.nan + else: + off_ratio = len(off_df) / len(calculated_values) + if off_ratio > calculation_tolerance: + raise AssertionError( + f"Calculations in {table_name} are off by {off_ratio:.2%}. Expected tolerance " + f"of {calculation_tolerance:.1%}." + ) + + # We'll only get here if the proportion of calculations that are off is acceptable + if (off_ratio > 0 or np.isnan(off_ratio)) and add_corrections: + logger.info( + f"{table_name}: has {len(off_df)} ({off_ratio:.02%}) records whose " + "calculations don't match. Adding correction records to make calculations " + "match reported values." + ) + corrections = off_df.copy() + corrections[value_col] = ( + corrections[value_col].fillna(0.0) - corrections["calculated_amount"] + ) + corrections["original_factoid"] = corrections["xbrl_factoid"] + corrections["xbrl_factoid"] = corrections["xbrl_factoid"] + "_correction" + corrections["row_type_xbrl"] = "correction" + corrections["is_within_table_calc"] = False + corrections["record_id"] = pd.NA + + calculated_df = pd.concat( + [calculated_df, corrections], axis="index" + ).reset_index() + return calculated_df + + class Ferc1TableTransformParams(TableTransformParams): """A model defining what TransformParams are allowed for FERC Form 1. @@ -2365,6 +2394,7 @@ def reconcile_table_calculations( xbrl_factoid_name=self.params.xbrl_factoid_name, table_name=self.table_id.value, params=params, + add_corrections=True, ) return df diff --git a/test/unit/transform/ferc1_test.py b/test/unit/transform/ferc1_test.py index b366a4ceb4..9dde46fd10 100644 --- a/test/unit/transform/ferc1_test.py +++ b/test/unit/transform/ferc1_test.py @@ -451,7 +451,6 @@ def test_calculate_values_from_components(): out_ksr = calculate_values_from_components( calculation_components=calculation_components_ksr, data=data_ksr, - validate="one_to_many", calc_idx=["table_name", "xbrl_factoid", "planet"], value_col="value", )