diff --git a/.github/workflows/zenodo-cache-sync.yml b/.github/workflows/zenodo-cache-sync.yml index 7bb4555c27..18d8575d52 100644 --- a/.github/workflows/zenodo-cache-sync.yml +++ b/.github/workflows/zenodo-cache-sync.yml @@ -37,12 +37,15 @@ jobs: ref: ${{ env.GITHUB_REF }} - name: Install Conda environment using mamba - uses: mamba-org/provision-with-micromamba@v16 + uses: mamba-org/setup-micromamba@v1 with: environment-file: test/test-environment.yml - cache-env: true - channels: conda-forge,defaults - channel-priority: strict + cache-environment: true + condarc: | + channels: + - conda-forge + - defaults + channel_priority: strict - name: Log environment details run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2cbb858aea..18eecbaf2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,6 @@ repos: - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - - id: python-check-blanket-noqa # Prohibit overly broad QA exclusions. - id: rst-backticks # Find single rather than double backticks - id: rst-directive-colons # Missing double-colons after directives - id: rst-inline-touching-normal # Inline code should never touch normal text @@ -27,7 +26,7 @@ repos: # Formatters: hooks that re-write Python & documentation files #################################################################################### - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.288 + rev: v0.0.290 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/src/pudl/convert/censusdp1tract_to_sqlite.py b/src/pudl/convert/censusdp1tract_to_sqlite.py index 617f529a17..81dd0ce399 100644 --- a/src/pudl/convert/censusdp1tract_to_sqlite.py +++ b/src/pudl/convert/censusdp1tract_to_sqlite.py @@ -63,7 +63,7 @@ def censusdp1tract_to_sqlite(context): # fragile solution that will not work on all platforms, but should cover # conda environments, Docker, and continuous integration on GitHub. ogr2ogr = Path(os.environ.get("CONDA_PREFIX", "/usr")) / "bin/ogr2ogr" - assert ogr2ogr.exists() + assert ogr2ogr.is_file() # Extract the sippzed GeoDB archive from the Datastore into a temporary # directory so that ogr2ogr can operate on it. Output the resulting SQLite # database into the user's PUDL workspace. We do not need to keep the diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 1b813d42cb..9ff916d221 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -1,7 +1,9 @@ """A collection of denormalized FERC assets and helper functions.""" import importlib import re -from typing import Literal, NamedTuple, Self +from copy import deepcopy +from functools import cached_property +from typing import Any, Literal, NamedTuple, Self import networkx as nx import numpy as np @@ -10,13 +12,46 @@ from matplotlib import pyplot as plt from networkx.drawing.nx_agraph import graphviz_layout from pandas._libs.missing import NAType as pandas_NAType -from pydantic import BaseModel, validator +from pydantic import BaseModel, confloat, validator import pudl logger = pudl.logging_helpers.get_logger(__name__) +class CalculationToleranceFerc1(BaseModel): + """Data quality expectations related to FERC 1 calculations. + + We are doing a lot of comparisons between calculated and reported values to identify + reporting errors in the data, errors in FERC's metadata, and bugs in our own code. + This class provides a structure for encoding our expectations about the level of + acceptable (or at least expected) errors, and allows us to pass them around. + + In the future we might also want to specify much more granular expectations, + pertaining to individual tables, years, utilities, or facts to ensure that we don't + have low overall error rates, but a problem with the way the data or metadata is + reported in a particular year. We could also define per-filing and per-table error + tolerances to help us identify individual utilities that have e.g. used an outdated + version of Form 1 when filing. + """ + + intertable_calculation_errors: confloat(ge=0.0, le=1.0) = 0.05 + """Fraction of interatble calculations that are allowed to not match exactly.""" + + +EXPLOSION_CALCULATION_TOLERANCES: dict[str, CalculationToleranceFerc1] = { + "income_statement_ferc1": CalculationToleranceFerc1( + intertable_calculation_errors=0.20, + ), + "balance_sheet_assets_ferc1": CalculationToleranceFerc1( + intertable_calculation_errors=0.85, + ), + "balance_sheet_liabilities_ferc1": CalculationToleranceFerc1( + intertable_calculation_errors=0.07, + ), +} + + @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") def denorm_plants_utilities_ferc1( plants_ferc1: pd.DataFrame, @@ -906,12 +941,46 @@ class NodeId(NamedTuple): plant_function: str | pandas_NAType +@asset +def _out_ferc1__explosion_tags(table_dimensions_ferc1) -> pd.DataFrame: + """Grab the stored table of tags and add infered dimension.""" + # NOTE: there are a bunch of duplicate records in xbrl_factoid_rate_base_tags.csv + # Also, these tags are only applicable to the balance_sheet_assets_ferc1 table, but + # we need to pass in a dataframe with the right structure to all of the exploders, + # so we're just re-using this one for the moment. + tags_csv = ( + importlib.resources.files("pudl.package_data.ferc1") + / "xbrl_factoid_rate_base_tags.csv" + ) + tags_df = ( + pd.read_csv( + tags_csv, + usecols=[ + "table_name", + "xbrl_factoid", + "in_rate_base", + "utility_type", + "plant_function", + "plant_status", + ], + ) + .drop_duplicates() + .dropna(subset=["table_name", "xbrl_factoid"], how="any") + .pipe( + pudl.transform.ferc1.make_calculation_dimensions_explicit, + table_dimensions_ferc1, + dimensions=["utility_type", "plant_function", "plant_status"], + ) + .astype(pd.StringDtype()) + ) + return tags_df + + def exploded_table_asset_factory( root_table: str, table_names_to_explode: list[str], - tags: pd.DataFrame, - seed_nodes: list[NodeId] = [], - calculation_tolerance: float = 0.05, + seed_nodes: list[NodeId], + calculation_tolerance: CalculationToleranceFerc1, io_manager_key: str | None = None, ) -> AssetsDefinition: """Create an exploded table based on a set of related input tables.""" @@ -920,6 +989,7 @@ def exploded_table_asset_factory( "calculation_components_xbrl_ferc1": AssetIn( "calculation_components_xbrl_ferc1" ), + "_out_ferc1__explosion_tags": AssetIn("_out_ferc1__explosion_tags"), } ins |= {table_name: AssetIn(table_name) for table_name in table_names_to_explode} @@ -929,10 +999,16 @@ def exploded_tables_asset( ) -> pd.DataFrame: metadata_xbrl_ferc1 = kwargs["metadata_xbrl_ferc1"] calculation_components_xbrl_ferc1 = kwargs["calculation_components_xbrl_ferc1"] + tags = kwargs["_out_ferc1__explosion_tags"] tables_to_explode = { name: df for (name, df) in kwargs.items() - if name not in ["metadata_xbrl_ferc1", "calculation_components_xbrl_ferc1"] + if name + not in [ + "metadata_xbrl_ferc1", + "calculation_components_xbrl_ferc1", + "_out_ferc1__explosion_tags", + ] } return Exploder( table_names=tables_to_explode.keys(), @@ -941,10 +1017,8 @@ def exploded_tables_asset( calculation_components_xbrl_ferc1=calculation_components_xbrl_ferc1, seed_nodes=seed_nodes, tags=tags, - ).boom( - tables_to_explode=tables_to_explode, calculation_tolerance=calculation_tolerance, - ) + ).boom(tables_to_explode=tables_to_explode) return exploded_tables_asset @@ -956,21 +1030,6 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: A list of :class:`AssetsDefinitions` where each asset is an exploded FERC Form 1 table. """ - # NOTE: there are a bunch of duplicate records in xbrl_factoid_rate_base_tags.csv - # Also, these tags are only applicable to the balance_sheet_assets_ferc1 table, but - # we need to pass in a dataframe with the right structure to all of the exploders, - # so we're just re-using this one for the moment. - tags_csv = ( - importlib.resources.files("pudl.package_data.ferc1") - / "xbrl_factoid_rate_base_tags.csv" - ) - tags_df = ( - pd.read_csv(tags_csv, usecols=["table_name", "xbrl_factoid", "in_rate_base"]) - .drop_duplicates() - .dropna(subset=["table_name", "xbrl_factoid"], how="any") - .astype(pd.StringDtype()) - ) - explosion_args = [ { "root_table": "income_statement_ferc1", @@ -980,8 +1039,9 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: "electric_operating_expenses_ferc1", "electric_operating_revenues_ferc1", ], - # This is very high, otherwise CI w/ 2 years of data currently fails. - "calculation_tolerance": 0.28, + "calculation_tolerance": EXPLOSION_CALCULATION_TOLERANCES[ + "income_statement_ferc1" + ], "seed_nodes": [ NodeId( table_name="income_statement_ferc1", @@ -991,7 +1051,6 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: plant_function=pd.NA, ), ], - "tags": tags_df, }, { "root_table": "balance_sheet_assets_ferc1", @@ -1002,7 +1061,9 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: "plant_in_service_ferc1", "electric_plant_depreciation_functional_ferc1", ], - "calculation_tolerance": 0.81, + "calculation_tolerance": EXPLOSION_CALCULATION_TOLERANCES[ + "balance_sheet_assets_ferc1" + ], "seed_nodes": [ NodeId( table_name="balance_sheet_assets_ferc1", @@ -1012,7 +1073,6 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: plant_function=pd.NA, ) ], - "tags": tags_df, }, { "root_table": "balance_sheet_liabilities_ferc1", @@ -1021,7 +1081,9 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: "balance_sheet_liabilities_ferc1", "retained_earnings_ferc1", ], - "calculation_tolerance": 0.075, + "calculation_tolerance": EXPLOSION_CALCULATION_TOLERANCES[ + "balance_sheet_liabilities_ferc1" + ], "seed_nodes": [ NodeId( table_name="balance_sheet_liabilities_ferc1", @@ -1031,7 +1093,6 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: plant_function=pd.NA, ) ], - "tags": tags_df, }, ] return [exploded_table_asset_factory(**kwargs) for kwargs in explosion_args] @@ -1040,22 +1101,39 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: exploded_ferc1_assets = create_exploded_table_assets() -class MetadataExploder: - """Combine a set of inter-related, nested table's metadata.""" +class Exploder: + """Get unique, granular datapoints from a set of related, nested FERC1 tables.""" def __init__( - self, + self: Self, table_names: list[str], + root_table: str, metadata_xbrl_ferc1: pd.DataFrame, calculation_components_xbrl_ferc1: pd.DataFrame, + seed_nodes: list[NodeId], + tags: pd.DataFrame = pd.DataFrame(), + calculation_tolerance: CalculationToleranceFerc1 = CalculationToleranceFerc1(), ): - """Instantiate MetadataExploder.""" - self.table_names = table_names - self.calculation_components_xbrl_ferc1 = calculation_components_xbrl_ferc1 + """Instantiate an Exploder class. + + Args: + table_names: list of table names to explode. + root_table: the table at the base of the tree of tables_to_explode. + metadata_xbrl_ferc1: table of factoid-level metadata. + calculation_components_xbrl_ferc1: table of calculation components. + seed_nodes: NodeIds to use as seeds for the calculation forest. + tags: Additional metadata to merge onto the exploded dataframe. + """ + self.table_names: list[str] = table_names + self.root_table: str = root_table + self.calculation_tolerance = calculation_tolerance self.metadata_xbrl_ferc1 = metadata_xbrl_ferc1 + self.calculation_components_xbrl_ferc1 = calculation_components_xbrl_ferc1 + self.seed_nodes = seed_nodes + self.tags = tags - @property - def calculations(self: Self): + @cached_property + def exploded_calcs(self: Self): """Remove any calculation components that aren't relevant to the explosion. At the end of this process several things should be true: @@ -1132,22 +1210,19 @@ def calculations(self: Self): return calc_explode - @property - def metadata(self): + @cached_property + def exploded_meta(self: Self) -> pd.DataFrame: """Combine a set of interrelated table's metatada for use in :class:`Exploder`. Any calculations containing components that are part of tables outside the set of exploded tables will be converted to reported values with an empty calculation. Then we verify that all referenced calculation components actually appear as their own records within the concatenated metadata dataframe. - - Args: - clean_xbrl_metadata_json: cleaned XRBL metadata. """ calc_cols = list(NodeId._fields) exploded_metadata = ( self.metadata_xbrl_ferc1[ - self.metadata_xbrl_ferc1.table_name.isin(self.table_names) + self.metadata_xbrl_ferc1["table_name"].isin(self.table_names) ] .set_index(calc_cols) .sort_index() @@ -1155,83 +1230,85 @@ def metadata(self): ) # At this point all remaining calculation components should exist within the # exploded metadata. - calc_comps = self.calculations + calc_comps = self.exploded_calcs missing_from_calcs_idx = calc_comps.set_index(calc_cols).index.difference( calc_comps.set_index(calc_cols).index ) assert missing_from_calcs_idx.empty - return exploded_metadata + # Add additional metadata useful for debugging calculations and tagging: + def snake_to_camel_case(factoid: str): + return "".join([word.capitalize() for word in factoid.split("_")]) -class Exploder: - """Get unique, granular datapoints from a set of related, nested FERC1 tables.""" + exploded_metadata["xbrl_taxonomy_fact_name"] = exploded_metadata[ + "xbrl_factoid_original" + ].apply(snake_to_camel_case) + pudl_to_xbrl_map = { + pudl_table: source_tables["xbrl"] + for pudl_table, source_tables in pudl.extract.ferc1.TABLE_NAME_MAP_FERC1.items() + } + exploded_metadata["xbrl_schedule_name"] = exploded_metadata["table_name"].map( + pudl_to_xbrl_map + ) + + def get_dbf_row_metadata(pudl_table: str, year: int = 2020): + dbf_tables = pudl.transform.ferc1.FERC1_TFR_CLASSES[ + pudl_table + ]().params.aligned_dbf_table_names + dbf_metadata = ( + pudl.transform.ferc1.read_dbf_to_xbrl_map(dbf_table_names=dbf_tables) + .pipe(pudl.transform.ferc1.fill_dbf_to_xbrl_map) + .query("report_year==@year") + .drop(columns="report_year") + .astype( + { + "row_number": "Int64", + "row_literal": "string", + } + ) + .rename( + columns={ + "row_number": f"dbf{year}_row_number", + "row_literal": f"dbf{year}_row_literal", + "sched_table_name": f"dbf{year}_table_name", + } + ) + .assign(table_name=pudl_table) + .drop_duplicates(subset=["table_name", "xbrl_factoid"]) + ) + return dbf_metadata - def __init__( - self: Self, - table_names: list[str], - root_table: str, - metadata_xbrl_ferc1: pd.DataFrame, - calculation_components_xbrl_ferc1: pd.DataFrame, - seed_nodes: list[NodeId] = [], - tags: pd.DataFrame = pd.DataFrame(), - ): - """Instantiate an Exploder class. + dbf_row_metadata = pd.concat( + [get_dbf_row_metadata(table) for table in self.table_names] + ) - Args: - table_names: list of table names to explode. - root_table: the table at the base of the tree of tables_to_explode. - metadata_xbrl_ferc1: table of factoid-level metadata. - calculation_components_xbrl_ferc1: table of calculation components. - seed_nodes: NodeIds to use as seeds for the calculation forest. - tags: Additional metadata to merge onto the exploded dataframe. - """ - self.table_names: list[str] = table_names - self.root_table: str = root_table - self.meta_exploder = MetadataExploder( - self.table_names, - metadata_xbrl_ferc1, - calculation_components_xbrl_ferc1, + exploded_metadata = exploded_metadata.merge( + dbf_row_metadata, + how="left", + on=["table_name", "xbrl_factoid"], + validate="many_to_one", ) - self.metadata_exploded: pd.DataFrame = self.meta_exploder.metadata - self.calculations_exploded: pd.DataFrame = self.meta_exploder.calculations - # If we don't get any explicit seed nodes, use all nodes from the root table - # that have calculations associated with them: - if len(seed_nodes) == 0: - logger.info( - "No seeds provided. Using all calculated nodes in root table: " - f"{self.root_table}" - ) - seed_nodes = [ - NodeId(seed) - for seed in self.metadata_exploded[ - (self.metadata_exploded.table_name == self.root_table) - & (self.metadata_exploded.calculations != "[]") - ] - .set_index(["table_name", "xbrl_factoid"]) - .index - ] - logger.info(f"Identified {seed_nodes=}") - self.seed_nodes = seed_nodes - self.tags = tags + return exploded_metadata - @property + @cached_property def calculation_forest(self: Self) -> "XbrlCalculationForestFerc1": """Construct a calculation forest based on class attributes.""" return XbrlCalculationForestFerc1( - exploded_calcs=self.calculations_exploded, - exploded_meta=self.metadata_exploded, + exploded_calcs=self.exploded_calcs, + exploded_meta=self.exploded_meta, seeds=self.seed_nodes, tags=self.tags, + calculation_tolerance=self.calculation_tolerance, ) - @property - def other_dimensions(self) -> list[str]: + @cached_property + def other_dimensions(self: Self) -> list[str]: """Get all of the column names for the other dimensions.""" return pudl.transform.ferc1.other_dimensions(table_names=self.table_names) - @property - def exploded_pks(self) -> list[str]: + @cached_property + def exploded_pks(self: Self) -> list[str]: """Get the joint primary keys of the exploded tables.""" pks = [] for table_name in self.table_names: @@ -1255,8 +1332,8 @@ def exploded_pks(self) -> list[str]: ] + pudl.helpers.dedupe_n_flatten_list_of_lists(pks) return pks - @property - def value_col(self) -> str: + @cached_property + def value_col(self: Self) -> str: """Get the value column for the exploded tables.""" value_cols = [] for table_name in self.table_names: @@ -1273,11 +1350,7 @@ def value_col(self) -> str: value_col = list(set(value_cols))[0] return value_col - def boom( - self, - tables_to_explode: dict[str, pd.DataFrame], - calculation_tolerance: float = 0.05, - ) -> pd.DataFrame: + def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame: """Explode a set of nested tables. There are five main stages of this process: @@ -1296,7 +1369,10 @@ def boom( exploded = ( self.initial_explosion_concatenation(tables_to_explode) .pipe(self.generate_intertable_calculations) - .pipe(self.reconcile_intertable_calculations, calculation_tolerance) + .pipe( + self.reconcile_intertable_calculations, + self.calculation_tolerance.intertable_calculation_errors, + ) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... @@ -1348,22 +1424,22 @@ def initial_explosion_concatenation( meta_idx = list(NodeId._fields) missing_dims = list(set(meta_idx).difference(exploded.columns)) # Missing dimensions SHOULD be entirely null in the metadata, if so we can drop - if not self.metadata_exploded.loc[:, missing_dims].isna().all(axis=None): + if not self.exploded_meta.loc[:, missing_dims].isna().all(axis=None): raise AssertionError( f"Expected missing metadata dimensions {missing_dims} to be null." ) - metadata_exploded = self.metadata_exploded.drop(columns=missing_dims) + exploded_meta = self.exploded_meta.drop(columns=missing_dims) meta_idx = list(set(meta_idx).difference(missing_dims)) # drop any metadata columns that appear in the data tables, because we may have # edited them in the metadata table, and want the edited version to take # precedence cols_to_keep = list( - set(exploded.columns).difference(metadata_exploded.columns).union(meta_idx) + set(exploded.columns).difference(exploded_meta.columns).union(meta_idx) ) exploded = pd.merge( left=exploded.loc[:, cols_to_keep], - right=metadata_exploded, + right=exploded_meta, how="left", on=meta_idx, validate="m:1", @@ -1384,8 +1460,8 @@ def generate_intertable_calculations( Args: exploded: concatenated tables for table explosion. """ - calculations_intertable = self.calculations_exploded[ - ~self.calculations_exploded.is_within_table_calc + calculations_intertable = self.exploded_calcs[ + ~self.exploded_calcs.is_within_table_calc ] if calculations_intertable.empty: return exploded @@ -1473,7 +1549,7 @@ def reconcile_intertable_calculations( Args: calculated_df: table with calculated fields calculation_tolerance: What proportion (0-1) of calculated values are - allowed to be incorrect without raising an AssertionError. + allowed to be incorrect without raising an AssertionError. """ if "calculated_amount" not in calculated_df.columns: return calculated_df @@ -1567,11 +1643,11 @@ class XbrlCalculationForestFerc1(BaseModel): several trees) rather than a single tree. The information required to build a calculation forest is most readily found in the - data produced by :meth:`MetadataExploder.boom` A list of seed nodes can also be - supplied, indicating which nodes must be present in the resulting forest. This can - be used to prune irrelevant portions of the overall forest out of the exploded - metadata. If no seeds are provided, then all of the nodes referenced in the - exploded_calcs input dataframe will be used as seeds. + :meth:`Exploder.exploded_calcs` A list of seed nodes can also be supplied, + indicating which nodes must be present in the resulting forest. This can be used to + prune irrelevant portions of the overall forest out of the exploded metadata. If no + seeds are provided, then all of the nodes referenced in the exploded_calcs input + dataframe will be used as seeds. This class makes heavy use of :mod:`networkx` to manage the graph that we build from calculation relationships. @@ -1584,11 +1660,13 @@ class XbrlCalculationForestFerc1(BaseModel): exploded_calcs: pd.DataFrame = pd.DataFrame() seeds: list[NodeId] = [] tags: pd.DataFrame = pd.DataFrame() + calculation_tolerance: CalculationToleranceFerc1 = CalculationToleranceFerc1() class Config: """Allow the class to store a dataframe.""" arbitrary_types_allowed = True + keep_untouched = (cached_property,) @validator("parent_cols", always=True) def set_parent_cols(cls, v, values) -> list[str]: @@ -1619,12 +1697,9 @@ def single_valued_weights(cls, v: pd.DataFrame, values) -> pd.DataFrame: .gt(1) ) if multi_valued_weights.any(): - # Maybe this should be an AssertionError but we have one weird special case - # That we are dealing with explicitly in building the trees below. Maybe it - # should be happening here instead? logger.warning( - "Calculation forest nodes specified with conflicting weights:\n" - f"{v.loc[multi_valued_weights]}" + f"Found {sum(multi_valued_weights)} calculations with conflicting " + "weights." ) return v @@ -1649,9 +1724,7 @@ def calc_parents_notna(cls, v: pd.DataFrame) -> pd.DataFrame: @validator("tags") def tags_have_required_cols(cls, v: pd.DataFrame, values) -> pd.DataFrame: """Ensure tagging dataframe contains all required index columns.""" - missing_cols = [ - col for col in ["table_name", "xbrl_factoid"] if col not in v.columns - ] + missing_cols = [col for col in values["calc_cols"] if col not in v.columns] if missing_cols: raise ValueError( f"Tagging dataframe was missing expected columns: {missing_cols=}" @@ -1674,7 +1747,7 @@ def tags_cols_notnull(cls, v: pd.DataFrame) -> pd.DataFrame: @validator("tags") def single_valued_tags(cls, v: pd.DataFrame, values) -> pd.DataFrame: """Ensure all tags have unique values.""" - dupes = v.duplicated(subset=["table_name", "xbrl_factoid"], keep=False) + dupes = v.duplicated(subset=values["calc_cols"], keep=False) if dupes.any(): logger.warning( f"Found {dupes.sum()} duplicate tag records:\n{v.loc[dupes]}" @@ -1695,15 +1768,6 @@ def seeds_within_bounds(cls, v: pd.DataFrame, values) -> pd.DataFrame: raise ValueError(f"Seeds missing from exploded_calcs index: {bad_seeds=}") return v - # Need to update this to generate a new valid set of seeds - # @validator("seeds", always=True) - # def seeds_not_empty(cls, v, values): - # """If no seeds are provided, use all nodes in the index of exploded_calcs.""" - # if v == []: - # logger.info("No seeds provided. Using all nodes from exploded_calcs.") - # v = list(values["exploded_calcs"].index) - # return v - def exploded_calcs_to_digraph( self: Self, exploded_calcs: pd.DataFrame, @@ -1731,68 +1795,135 @@ def exploded_calcs_to_digraph( forest = nx.from_pandas_edgelist(edgelist, create_using=nx.DiGraph) return forest - def set_forest_attributes( - self, - forest: nx.DiGraph, - exploded_calcs: pd.DataFrame, - exploded_meta: pd.DataFrame, - tags: pd.DataFrame, - ) -> nx.DiGraph: - """Set the attributes of a forest.""" + @cached_property + def node_attrs(self: Self) -> dict[NodeId, dict[str, dict[str, str]]]: + """Construct a dictionary of node attributes for application to the forest. + + Note attributes consist of the manually assigned tags. + """ # Reshape the tags to turn them into a dictionary of values per-node. This # will make it easier to add arbitrary sets of tags later on. - tags_dict = tags.set_index(["table_name", "xbrl_factoid"]).to_dict( - orient="index" + tags_dict = ( + self.tags.convert_dtypes().set_index(self.calc_cols).to_dict(orient="index") ) - tags_dict_df = pd.DataFrame( - index=pd.MultiIndex.from_tuples( - tags_dict.keys(), names=["table_name", "xbrl_factoid"] - ), - data={"tags": list(tags_dict.values())}, - ).reset_index() - - multi_valued_weights = ( - exploded_calcs.groupby(self.calc_cols, dropna=False)["weight"] - .transform("nunique") - .gt(1) - ) - - calcs_to_drop = multi_valued_weights & (exploded_calcs.weight == 1) - - logger.info( - f"Found {len(calcs_to_drop)}/{len(exploded_calcs)} calculations " - "where weight was both 1 and -1. Dropping the 1-weighted " - "calculations." + node_attrs = ( + pd.DataFrame( + index=pd.MultiIndex.from_tuples(tags_dict.keys(), names=self.calc_cols), + data={"tags": list(tags_dict.values())}, + ) + .reset_index() + # Type conversion is necessary to get pd.NA in the index: + .astype({col: pd.StringDtype() for col in self.calc_cols}) + # We need a dictionary for *all* nodes, not just those with tags. + .merge( + self.exploded_meta.loc[:, self.calc_cols], + how="right", + on=self.calc_cols, + validate="one_to_many", + ) + # For nodes with no tags, we assign an empty dictionary: + .assign(tags=lambda x: np.where(x["tags"].isna(), {}, x["tags"])) + .set_index(self.calc_cols) + .to_dict(orient="index") ) + return node_attrs - attr_cols = ["weight"] - deduped_calcs = exploded_calcs[~calcs_to_drop][ - self.calc_cols + attr_cols - ].drop_duplicates() + @cached_property + def edge_attrs(self: Self) -> dict[Any, Any]: + """Construct a dictionary of edge attributes for application to the forest. - meta_w_tags = ( - pd.merge( - left=exploded_meta, - right=tags_dict_df, - how="left", - validate="m:1", - ) - .reset_index(drop=True) - .drop(columns=["xbrl_factoid_original", "is_within_table_calc"]) - ) - # Add metadata tags to the calculation components and reset the index. - node_attrs = pd.merge( - left=meta_w_tags, - right=deduped_calcs, - how="left", - validate="1:1", - ).set_index(self.calc_cols) - # Fill NA tag dictionaries with an empty dict so the type is uniform: - node_attrs["tags"] = node_attrs["tags"].apply(lambda x: {} if x != x else x) - nx.set_node_attributes(forest, node_attrs.to_dict(orient="index")) - return forest + The only edge attribute is the calculation component weight. + """ + parents = [ + NodeId(*x) + for x in self.exploded_calcs.set_index(self.parent_cols).index.to_list() + ] + children = [ + NodeId(*x) + for x in self.exploded_calcs.set_index(self.calc_cols).index.to_list() + ] + weights = self.exploded_calcs["weight"].to_list() + edge_attrs = { + (parent, child): {"weight": weight} + for parent, child, weight in zip(parents, children, weights) + } + return edge_attrs + + @cached_property + def annotated_forest(self: Self) -> nx.DiGraph: + """Annotate the calculation forest with node calculation weights and tags. + + The annotated forest should have exactly the same structure as the forest, but + with additional data associated with each of the nodes. This method also does + some error checking to try and ensure that the weights and tags that are being + associated with the forest are internally self-consistent. + + We check whether there are multiple different weights assocated with the same + node in the calculation components. There are a few instances where this is + expected, but if there a lot of conflicting weights something is probably wrong. + + We check whether any of the nodes that were orphaned (never connected to the + graph) or that were pruned in the course of enforcing a forest structure had + manually assigned tags (e.g. indicating whether they contribute to rate base). + If they do, then the final exploded data table may not capture all of the + manually assigned metadata, and we either need to edit the metadata, or figure + out why those nodes aren't being included in the final calculation forest. + """ + annotated_forest = deepcopy(self.forest) + nx.set_node_attributes(annotated_forest, self.node_attrs) + nx.set_edge_attributes(annotated_forest, self.edge_attrs) + + logger.info("Checking whether any pruned nodes were also tagged.") + self.check_lost_tags(lost_nodes=self.pruned) + logger.info("Checking whether any orphaned nodes were also tagged.") + self.check_lost_tags(lost_nodes=self.orphans) + self.check_conflicting_tags(annotated_forest) + return annotated_forest + + def check_lost_tags(self: Self, lost_nodes: list[NodeId]) -> None: + """Check whether any of the input lost nodes were also tagged nodes.""" + if lost_nodes: + lost = pd.DataFrame(lost_nodes).set_index(self.calc_cols) + tagged = self.tags.set_index(self.calc_cols) + lost_tagged = tagged.index.intersection(lost.index) + if not lost_tagged.empty: + logger.warning( + "The following tagged nodes were lost in building the forest:\n" + f"{tagged.loc[lost_tagged].sort_index()}" + ) - @property + @staticmethod + def check_conflicting_tags(annotated_forest: nx.DiGraph) -> None: + """Check for conflicts between ancestor and descendant tags. + + At this point, we have just applied the manually compiled tags to the nodes in + the forest, and haven't yet propagated them down to the leaves. It's possible + that ancestor nodes (closer to the roots) might have tags associated with them + that are in conflict with descendant nodes (closer to the leaves). If that's + the case then when we propagate the tags to the leaves, whichever tag is + propagated last will end up taking precedence. + + These kinds of conflicts are probably due to errors in the tagging metadata, and + should be investigated. + """ + nodes = annotated_forest.nodes + for ancestor in nodes: + for descendant in nx.descendants(annotated_forest, ancestor): + for tag in nodes[ancestor]["tags"]: + if tag in nodes[descendant]["tags"]: + ancestor_tag_value = nodes[ancestor]["tags"][tag] + descendant_tag_value = nodes[descendant]["tags"][tag] + if ancestor_tag_value != descendant_tag_value: + logger.error( + "\n================================================" + f"\nCalculation forest nodes have conflicting tags:" + f"\nAncestor: {ancestor}" + f"\n tags[{tag}] == {ancestor_tag_value}" + f"\nDescendant: {descendant}" + f"\n tags[{tag}] == {descendant_tag_value}" + ) + + @cached_property def full_digraph(self: Self) -> nx.DiGraph: """A digraph of all calculations described by the exploded metadata.""" full_digraph = self.exploded_calcs_to_digraph( @@ -1811,28 +1942,50 @@ def full_digraph(self: Self) -> nx.DiGraph: return full_digraph def prune_unrooted(self: Self, graph: nx.DiGraph) -> nx.DiGraph: - """Prune any portions of the digraph that aren't reachable from the roots.""" + """Prune those parts of the input graph that aren't reachable from the roots. + + Build a table of exploded calculations that includes only those nodes that + are part of the input graph, and that are reachable from the roots of the + calculation forest. Then use that set of exploded calculations to construct a + new graph. + + This is complicated by the fact that some nodes may have already been pruned + from the input graph, and so when selecting both parent and child nodes from + the calculations, we need to make sure that they are present in the input graph, + as well as the complete set of calculation components. + """ seeded_nodes = set(self.seeds) for seed in self.seeds: + # the seeds and all of their descendants from the graph seeded_nodes = list( seeded_nodes.union({seed}).union(nx.descendants(graph, seed)) ) + # Any seeded node that appears in the input graph and is also a parent. seeded_parents = [ node for node, degree in dict(graph.out_degree(seeded_nodes)).items() if degree > 0 ] + # Any calculation where the parent is one of the seeded parents. seeded_calcs = ( self.exploded_calcs.set_index(self.parent_cols) .loc[seeded_parents] .reset_index() ) - seeded_digraph: nx.DiGraph = self.exploded_calcs_to_digraph( - exploded_calcs=seeded_calcs + # All child nodes in the seeded calculations that are part of the input graph. + seeded_child_nodes = list( + set( + seeded_calcs[self.calc_cols].itertuples(index=False, name="NodeId") + ).intersection(graph.nodes) ) - return seeded_digraph + # This seeded calcs includes only calculations where both the parent and child + # nodes were part of the input graph. + seeded_calcs = ( + seeded_calcs.set_index(self.calc_cols).loc[seeded_child_nodes].reset_index() + ) + return self.exploded_calcs_to_digraph(exploded_calcs=seeded_calcs) - @property + @cached_property def seeded_digraph(self: Self) -> nx.DiGraph: """A digraph of all calculations that contribute to the seed values. @@ -1848,7 +2001,7 @@ def seeded_digraph(self: Self) -> nx.DiGraph: """ return self.prune_unrooted(self.full_digraph) - @property + @cached_property def forest(self: Self) -> nx.DiGraph: """A pruned version of the seeded digraph that should be one or more trees. @@ -1862,69 +2015,69 @@ def forest(self: Self) -> nx.DiGraph: table may or may not have a top level summary value that includes all underlying calculated values of interest. """ - forest = self.seeded_digraph - # Remove any node that has only one parent and one child, and add an edge - # between its parent and child. - for node in self.passthroughs: - parent = list(forest.predecessors(node)) - assert len(parent) == 1 - successors = forest.successors(node) - assert len(list(successors)) == 2 - child = [ - n - for n in forest.successors(node) - if not n.xbrl_factoid.endswith("_correction") - ] - correction = [ - n - for n in forest.successors(node) - if n.xbrl_factoid.endswith("_correction") - ] - assert len(child) == 1 - logger.debug( - f"Replacing passthrough node {node} with edge from " - f"{parent[0]} to {child[0]}" - ) - forest.remove_nodes_from(correction + [node]) - forest.add_edge(parent[0], child[0]) - - if not nx.is_forest(forest): - logger.error( - "Calculations in Exploded Metadata can not be represented as a forest!" - ) - connected_components = list(nx.connected_components(forest.to_undirected())) - logger.debug( - f"Calculation forest contains {len(connected_components)} connected components." - ) - - # Remove any node that: - # - ONLY has stepchildren. - # - AND has utility_type total - for node in self.stepparents(forest): + forest = deepcopy(self.seeded_digraph) + # Remove any node that ONLY has stepchildren. + # A stepparent is a node that has a child with more than one parent. + # A stepchild is a node with more than one parent. + # See self.stepparents and self.stepchildren + pure_stepparents = [] + stepparents = sorted(self.stepparents(forest)) + logger.info(f"Investigating {len(stepparents)=}") + for node in stepparents: children = set(forest.successors(node)) stepchildren = set(self.stepchildren(forest)).intersection(children) - if ( - (children == stepchildren) - & (len(children) > 0) - & (node.utility_type == "total") - ): + if (children == stepchildren) & (len(children) > 0): + pure_stepparents.append(node) forest.remove_node(node) + logger.info(f"Removed {len(pure_stepparents)} redundant/stepparent nodes.") + logger.debug(f"Removed redunant/stepparent nodes: {sorted(pure_stepparents)}") - # Prune any newly disconnected nodes resulting from the above removal of - # pure stepparents. We expect the set of newly disconnected nodes to be empty. + # Removing pure stepparents should NEVER disconnect nodes from the forest. + # Defensive check to ensure that this is actually true nodes_before_pruning = forest.nodes forest = self.prune_unrooted(forest) nodes_after_pruning = forest.nodes - if pruned_nodes := set(nodes_before_pruning).difference(nodes_after_pruning): raise AssertionError(f"Unexpectedly pruned stepchildren: {pruned_nodes=}") - forest = self.set_forest_attributes( - forest, - exploded_meta=self.exploded_meta, - exploded_calcs=self.exploded_calcs, - tags=self.tags, - ) + # HACK alter. + # two different parents. those parents have different sets of dimensions. + # sharing some but not all of their children so they weren't caught from in the + # only stepchildren node removal from above. a generalization here would be good + almost_pure_stepparents = [ + NodeId( + "utility_plant_summary_ferc1", + "depreciation_amortization_and_depletion_utility_plant_leased_to_others", + "total", + pd.NA, + pd.NA, + ), + NodeId( + "utility_plant_summary_ferc1", + "depreciation_and_amortization_utility_plant_held_for_future_use", + "total", + pd.NA, + pd.NA, + ), + NodeId( + "utility_plant_summary_ferc1", + "utility_plant_in_service_classified_and_unclassified", + "total", + pd.NA, + pd.NA, + ), + ] + forest.remove_nodes_from(almost_pure_stepparents) + + forest = self.prune_unrooted(forest) + if not nx.is_forest(forest): + logger.error( + "Calculations in Exploded Metadata can not be represented as a forest!" + ) + remaining_stepparents = set(self.stepparents(forest)) + if remaining_stepparents: + logger.error(f"{remaining_stepparents=}") + return forest @staticmethod @@ -1932,17 +2085,17 @@ def roots(graph: nx.DiGraph) -> list[NodeId]: """Identify all root nodes in a digraph.""" return [n for n, d in graph.in_degree() if d == 0] - @property + @cached_property def full_digraph_roots(self: Self) -> list[NodeId]: """Find all roots in the full digraph described by the exploded metadata.""" return self.roots(graph=self.full_digraph) - @property + @cached_property def seeded_digraph_roots(self: Self) -> list[NodeId]: """Find all roots in the seeded digraph.""" return self.roots(graph=self.seeded_digraph) - @property + @cached_property def forest_roots(self: Self) -> list[NodeId]: """Find all roots in the pruned calculation forest.""" return self.roots(graph=self.forest) @@ -1952,22 +2105,22 @@ def leaves(graph: nx.DiGraph) -> list[NodeId]: """Identify all leaf nodes in a digraph.""" return [n for n, d in graph.out_degree() if d == 0] - @property + @cached_property def full_digraph_leaves(self: Self) -> list[NodeId]: """All leaf nodes in the full digraph.""" return self.leaves(graph=self.full_digraph) - @property + @cached_property def seeded_digraph_leaves(self: Self) -> list[NodeId]: """All leaf nodes in the seeded digraph.""" return self.leaves(graph=self.seeded_digraph) - @property + @cached_property def forest_leaves(self: Self) -> list[NodeId]: """All leaf nodes in the pruned forest.""" return self.leaves(graph=self.forest) - @property + @cached_property def orphans(self: Self) -> list[NodeId]: """Identify all nodes that appear in metadata but not in the full digraph.""" nodes = self.full_digraph.nodes @@ -1977,7 +2130,7 @@ def orphans(self: Self) -> list[NodeId]: if n not in nodes ] - @property + @cached_property def pruned(self: Self) -> list[NodeId]: """List of all nodes that appear in the DAG but not in the pruned forest.""" return list(set(self.full_digraph.nodes).difference(self.forest.nodes)) @@ -1994,36 +2147,7 @@ def stepparents(self: Self, graph: nx.DiGraph) -> list[NodeId]: stepparents = stepparents.union(graph.predecessors(stepchild)) return list(stepparents) - @property - def passthroughs(self: Self) -> list[NodeId]: - """All nodes in the seeded digraph with a single parent and a single child. - - These nodes can be pruned, hopefully converting the seeded digraph into a - forest. Note that having a "single child" really means having 2 children, one - of which is a _correction to the calculation. We verify that the two children - are one real child node, and one appropriate correction. - """ - # In theory every node should have only one parent, but just to be safe, since - # that's not always true right now: - has_one_parent = {n for n, d in self.seeded_digraph.in_degree() if d == 1} - # Calculated fields always have both the reported child and a correction that - # we have added, so having "one" child really means having 2 successor nodes. - may_have_one_child: set[NodeId] = { - n for n, d in self.seeded_digraph.out_degree() if d == 2 - } - # Check that one of these successors is the correction. - has_one_child = [] - for node in may_have_one_child: - children: set[NodeId] = set(self.seeded_digraph.successors(node)) - for child in children: - if (node.table_name == child.table_name) and ( - child.xbrl_factoid == node.xbrl_factoid + "_correction" - ): - has_one_child.append(node) - - return list(has_one_parent.intersection(has_one_child)) - - @property + @cached_property def leafy_meta(self: Self) -> pd.DataFrame: """Identify leaf facts and compile their metadata. @@ -2042,51 +2166,50 @@ def leafy_meta(self: Self) -> pd.DataFrame: - The weight associated with the leaf, in relation to its root. """ # Construct a dataframe that links the leaf node IDs to their root nodes: - pruned_forest = self.forest leaves = self.forest_leaves roots = self.forest_roots leaf_to_root_map = { leaf: root for leaf in leaves for root in roots - if leaf in nx.descendants(pruned_forest, root) + if leaf in nx.descendants(self.annotated_forest, root) } leaves_df = pd.DataFrame(list(leaf_to_root_map.keys())) roots_df = pd.DataFrame(list(leaf_to_root_map.values())).rename( columns={col: col + "_root" for col in self.calc_cols} ) - leafy_meta = pd.concat([leaves_df, roots_df], axis="columns") + leafy_meta = pd.concat([roots_df, leaves_df], axis="columns") # Propagate tags and weights to leaf nodes leaf_rows = [] for leaf in leaves: leaf_tags = {} - leaf_weight = pruned_forest.nodes[leaf].get("weight", 1.0) - for node in nx.ancestors(pruned_forest, leaf): - # TODO: need to check that there are no conflicts between tags that are - # being propagated, e.g. if two different ancestors have been tagged - # rate_base: yes and rate_base: no. - leaf_tags |= pruned_forest.nodes[node]["tags"] - # Root nodes have no weight because they don't come from calculations - # We assign them a weight of 1.0 - # if not pruned_forest.nodes[node].get("weight", False): - if pd.isna(pruned_forest.nodes[node]["weight"]): - assert node in roots - node_weight = 1.0 - else: - node_weight = pruned_forest.nodes[node]["weight"] - leaf_weight *= node_weight + ancestors = list(nx.ancestors(self.annotated_forest, leaf)) + [leaf] + for node in ancestors: + leaf_tags |= self.annotated_forest.nodes[node]["tags"] + # Calculate the product of all edge weights in path from root to leaf + all_paths = list( + nx.all_simple_paths(self.annotated_forest, leaf_to_root_map[leaf], leaf) + ) + # In a forest there should only be one path from root to leaf + assert len(all_paths) == 1 + path = all_paths[0] + leaf_weight = 1.0 + for parent, child in zip(path, path[1:]): + leaf_weight *= self.annotated_forest.get_edge_data(parent, child)[ + "weight" + ] # Construct a dictionary describing the leaf node and convert it into a # single row DataFrame. This makes adding arbitrary tags easy. leaf_attrs = { - "weight": leaf_weight, - "tags": leaf_tags, "table_name": leaf.table_name, "xbrl_factoid": leaf.xbrl_factoid, "utility_type": leaf.utility_type, "plant_status": leaf.plant_status, "plant_function": leaf.plant_function, + "weight": leaf_weight, + "tags": leaf_tags, } leaf_rows.append(pd.json_normalize(leaf_attrs, sep="_")) @@ -2097,7 +2220,7 @@ def leafy_meta(self: Self) -> pd.DataFrame: .convert_dtypes() ) - @property + @cached_property def root_calculations(self: Self) -> pd.DataFrame: """Produce a calculation components dataframe containing only roots and leaves. @@ -2107,7 +2230,7 @@ def root_calculations(self: Self) -> pd.DataFrame: """ return self.leafy_meta.rename(columns=lambda x: re.sub("_root$", "_parent", x)) - @property + @cached_property def table_names(self: Self) -> list[str]: """Produce the list of tables involved in this explosion.""" return list(self.exploded_calcs["table_name_parent"].unique()) @@ -2175,3 +2298,97 @@ def leafy_data( # Scale the data column of interest: leafy_data[value_col] = leafy_data[value_col] * leafy_data["weight"] return leafy_data.reset_index(drop=True).convert_dtypes() + + @cached_property + def forest_as_table(self: Self) -> pd.DataFrame: + """Construct a tabular representation of the calculation forest. + + Each generation of nodes, starting with the root(s) of the calculation forest, + make up a set of columns in the table. Each set of columns is merged onto + """ + logger.info("Recursively building a tabular version of the calculation forest.") + # Identify all root nodes in the forest: + layer0_nodes = [n for n, d in self.annotated_forest.in_degree() if d == 0] + # Convert them into the first layer of the dataframe: + layer0_df = pd.DataFrame(layer0_nodes).rename(columns=lambda x: x + "_layer0") + + return ( + self._add_layers_to_forest_as_table(df=layer0_df) + .dropna(axis="columns", how="all") + .convert_dtypes() + ) + + def _add_layers_to_forest_as_table(self: Self, df: pd.DataFrame) -> pd.DataFrame: + """Recursively add additional layers of nodes from the forest to the table. + + Given a dataframe with one or more set of columns with names corresponding to + the components of a NodeId with suffixes of the form _layerN, identify the + children of the nodes in the set of columns with the largest N, and merge them + onto the table, recursively until there are no more children to add. Creating a + tabular representation of the calculation forest that can be inspected in Excel. + + Include inter-layer calculation weights and tags associated with the nodes pre + propagation. + """ + # Identify the last layer of nodes present in the input dataframe. + current_layer = df.rename( + columns=lambda x: int(re.sub(r"^.*_layer(\d+)$", r"\1", x)) + ).columns.max() + logger.info(f"{current_layer=}") + suffix = f"_layer{current_layer}" + parent_cols = [col + suffix for col in self.calc_cols] + # Identify the list of nodes that are part of that last layer: + parent_nodes = list( + df[parent_cols] + .drop_duplicates() + .dropna(how="all") + .rename(columns=lambda x: x.removesuffix(suffix)) + .itertuples(name="NodeId", index=False) + ) + + # Identify the successors (children), if any, of each node in the last layer: + successor_dfs = [] + for node in parent_nodes: + successor_nodes = list(self.forest.successors(node)) + # If this particular node has no successors, skip to the next one. + if not successor_nodes: + continue + # Convert the list of successor nodes into a dataframe with layer = n+1 + successor_df = nodes_to_df( + calc_forest=self.annotated_forest, nodes=successor_nodes + ).rename(columns=lambda x: x + f"_layer{current_layer + 1}") + # Add a set of parent columns that all have the same values so we can merge + # this onto the previous layer + successor_df[parent_cols] = node + successor_dfs.append(successor_df) + + # If any child nodes were found, merge them onto the input dataframe creating + # a new layer , and recurse: + if successor_dfs: + new_df = df.merge(pd.concat(successor_dfs), on=parent_cols, how="outer") + df = self._add_layers_to_forest_as_table(df=new_df) + + # If no child nodes were found return the dataframe terminating the recursion. + return df + + +def nodes_to_df(calc_forest: nx.DiGraph, nodes: list[NodeId]) -> pd.DataFrame: + """Construct a dataframe from a list of nodes, including their annotations. + + NodeIds that are not present in the calculation forest will be ignored. + + Args: + calc_forest: A calculation forest made of nodes with "weight" and "tags" data. + nodes: List of :class:`NodeId` values to extract from the calculation forest. + + Returns: + A tabular dataframe representation of the nodes, including their tags, extracted + from the calculation forest. + """ + node_dict = { + k: v for k, v in dict(calc_forest.nodes(data=True)).items() if k in nodes + } + index = pd.DataFrame(node_dict.keys()).astype("string") + data = pd.DataFrame(node_dict.values()) + tags = pd.json_normalize(data.tags).astype("string") + return pd.concat([index, tags], axis="columns") diff --git a/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv b/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv index 43eb599113..0ea8932c91 100644 --- a/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv +++ b/src/pudl/package_data/ferc1/xbrl_factoid_rate_base_tags.csv @@ -1,218 +1,221 @@ -xbrl_factoid_rmi,xbrl_factoid_rmi_renamed,xbrl_factoid,table_name,in_rate_base,notes -accrued_utility_revenues,accrued_utility_revenues,accrued_utility_revenues,balance_sheet_assets_ferc1,yes,"At the option of the utility, the estimated amount accrued for service rendered, but not billed at the end of any accounting period, may be included herein. In case accruals are made for unbilled revenues, they shall be made likewise for unbilled expenses, such as for the purchase of energy." -advances_for_gas,advances_for_gas,advances_for_gas,balance_sheet_assets_ferc1,yes,"4/13/21 - changed from ""non-electric"" to ""mixed"" - could be for gas power plants." -allowances,allowances,allowances,electric_operating_expenses_ferc1,yes, -cash,cash,cash,balance_sheet_assets_ferc1,yes, -customer_accounts_receivable,customer_accounts_receivable,customer_accounts_receivable,balance_sheet_assets_ferc1,yes,"service provided & billed/collected, asset earned on." -derivative_instrument_assets,derivative_instrument_assets,derivative_instrument_assets,balance_sheet_assets_ferc1,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -fuel_stock,fuel_stock,fuel_stock,balance_sheet_assets_ferc1,yes, -fuel_stock_expenses_undistributed,fuel_stock_expenses_undistributed,fuel_stock_expenses_undistributed,balance_sheet_assets_ferc1,yes, -interest_and_dividends_receivable,interest_and_dividends_receivable,interest_and_dividends_receivable,balance_sheet_assets_ferc1,no,not customer deposits or cash working capital -liquefied_natural_gas_stored_and_held_for_processing,liquefied_natural_gas_stored_and_held_for_processing,liquefied_natural_gas_stored_and_held_for_processing,balance_sheet_assets_ferc1,yes, -merchandise,merchandise,merchandise,balance_sheet_assets_ferc1,yes, -miscellaneous_current_and_accrued_assets,miscellaneous_current_and_accrued_assets,miscellaneous_current_and_accrued_assets,balance_sheet_assets_ferc1,yes,includes customer credits/refunds if ordered by regulatory body -noncurrent_portion_of_allowances,noncurrent_portion_of_allowances,noncurrent_portion_of_allowances,balance_sheet_assets_ferc1,yes, -notes_receivable,notes_receivable,notes_receivable,balance_sheet_assets_ferc1,yes, -notes_receivable_from_associated_companies,notes_receivable_from_associated_companies,notes_receivable_from_associated_companies,balance_sheet_assets_ferc1,no,Don't believe associated companies are involved in utility services? -nuclear_materials_held_for_sale,nuclear_materials_held_for_sale,nuclear_materials_held_for_sale,balance_sheet_assets_ferc1,yes, -other_accounts_receivable,other_accounts_receivable,other_accounts_receivable,balance_sheet_assets_ferc1,yes, -other_materials_and_supplies,other_materials_and_supplies,other_materials_and_supplies,balance_sheet_assets_ferc1,yes, -plant_materials_and_operating_supplies,plant_materials_and_operating_supplies,plant_materials_and_operating_supplies,balance_sheet_assets_ferc1,yes, -prepayments,prepayments,prepayments,balance_sheet_assets_ferc1,yes, -rents_receivable,rents_receivable,rents_receivable,balance_sheet_assets_ferc1,yes,asset of customers/ratepayers -special_deposits,special_deposits,special_deposits,balance_sheet_assets_ferc1,yes,customer deposits - restricted cash -stores_expense_undistributed,stores_expense_undistributed,stores_expense_undistributed,balance_sheet_assets_ferc1,yes, -temporary_cash_investments,temporary_cash_investments,temporary_cash_investments,balance_sheet_assets_ferc1,yes, -(less)_long_term_portion_of_derivative_instrument_liabilities,long_term_portion_of_derivative_instrument_liabilities,long_term_portion_of_derivative_instrument_liabilities,balance_sheet_liabilities_ferc1,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -(less)_long_term_portion_of_derivative_instrument_liabilities_hedges,long_term_portion_of_derivative_instrument_liabilities_hedges,long_term_portion_of_derivative_instrument_liabilities_hedges,balance_sheet_liabilities_ferc1,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -accounts_payable,accounts_payable,accounts_payable,balance_sheet_liabilities_ferc1,yes, -accounts_payable_to_associated_companies,accounts_payable_to_associated_companies,accounts_payable_to_associated_companies,balance_sheet_liabilities_ferc1,no,Don't believe associated companies are involved in utility services? -customer_deposits,customer_deposits,customer_deposits,balance_sheet_liabilities_ferc1,yes, -dividends_declared,dividends_declared,dividends_declared,balance_sheet_liabilities_ferc1,no, -interest_accrued,interest_accrued,interest_accrued,balance_sheet_liabilities_ferc1,no, -matured_interest,matured_interest,matured_interest,balance_sheet_liabilities_ferc1,no,not cash working capital -matured_long_term_debt,matured_long_term_debt,matured_long_term_debt,balance_sheet_liabilities_ferc1,no,not cash working capital -miscellaneous_current_and_accrued_liabilities,miscellaneous_current_and_accrued_liabilities,miscellaneous_current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,yes,includes customer credits/refunds if ordered by regulatory body -notes_payable,notes_payable,notes_payable,balance_sheet_liabilities_ferc1,yes, -notes_payable_to_associated_companies,notes_payable_to_associated_companies,notes_payable_to_associated_companies,balance_sheet_liabilities_ferc1,yes, -obligations_under_capital_leases_current,obligations_under_capital_leases_current,obligations_under_capital_leases_current,balance_sheet_liabilities_ferc1,yes, -tax_collections_payable,tax_collections_payable,tax_collections_payable,balance_sheet_liabilities_ferc1,yes, -taxes_accrued,taxes_accrued,taxes_accrued,balance_sheet_liabilities_ferc1,yes, -accumulated_deferred_income_taxes,accumulated_deferred_income_taxes,accumulated_deferred_income_taxes,balance_sheet_assets_ferc1,yes, -accumulated_deferred_investment_tax_credits,accumulated_deferred_investment_tax_credits,accumulated_deferred_investment_tax_credits,balance_sheet_liabilities_ferc1,yes, -customer_advances_for_construction,customer_advances_for_construction,customer_advances_for_construction,balance_sheet_liabilities_ferc1,yes, -deferred_gains_from_disposition_of_utility_plant,deferred_gains_from_disposition_of_utility_plant,deferred_gains_from_disposition_of_utility_plant,balance_sheet_liabilities_ferc1,yes, -other_deferred_credits,other_deferred_credits,other_deferred_credits,balance_sheet_liabilities_ferc1,yes,"Includes things such as ""PJM Transmission Cost Allocation"", ""Spent Nuclear Fuel - Other"", and similar items." -other_regulatory_liabilities,other_regulatory_liabilities,other_regulatory_liabilities,balance_sheet_liabilities_ferc1,yes, -clearing_accounts,clearing_accounts,clearing_accounts,balance_sheet_assets_ferc1,yes, -extraordinary_property_losses,extraordinary_property_losses,extraordinary_property_losses,balance_sheet_assets_ferc1,no, -miscellaneous_deferred_debits,miscellaneous_deferred_debits,miscellaneous_deferred_debits,balance_sheet_assets_ferc1,yes, -other_preliminary_survey_and_investigation_charges,other_preliminary_survey_and_investigation_charges,other_preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes, -other_regulatory_assets,other_regulatory_assets,other_regulatory_assets,balance_sheet_assets_ferc1,yes, -temporary_facilities,temporary_facilities,temporary_facilities,balance_sheet_assets_ferc1,no, -unrecovered_plant_and_regulatory_study_costs,unrecovered_plant_and_regulatory_study_costs,unrecovered_plant_and_regulatory_study_costs,balance_sheet_assets_ferc1,yes, -unrecovered_purchased_gas_costs,unrecovered_purchased_gas_costs,unrecovered_purchased_gas_costs,balance_sheet_assets_ferc1,yes, -advances_from_associated_companies,advances_from_associated_companies,advances_from_associated_companies,balance_sheet_liabilities_ferc1,no,Don't believe associated companies are involved in utility services? -bonds,bonds,bonds,balance_sheet_liabilities_ferc1,no, -other_long_term_debt,other_long_term_debt,other_long_term_debt,balance_sheet_liabilities_ferc1,no, -unamortized_premium_on_long_term_debt,unamortized_premium_on_long_term_debt,unamortized_premium_on_long_term_debt,balance_sheet_liabilities_ferc1,no, -nuclear_fuel_under_capital_leases,nuclear_fuel_under_capital_leases,nuclear_fuel_under_capital_leases,balance_sheet_assets_ferc1,yes, -abandonment_of_leases,abandonment_of_leases,abandonment_of_leases,utility_plant_summary_ferc1,yes, -amortization_of_plant_acquisition_adjustment,amortization_of_plant_acquisition_adjustment,amortization_of_plant_acquisition_adjustment,utility_plant_summary_ferc1,yes, -amortization,amortization_utility_plant_held_for_future_use,amortization_utility_plant_held_for_future_use,utility_plant_summary_ferc1,yes, -construction_work_in_progress,construction_work_in_progress,construction_work_in_progress,balance_sheet_assets_ferc1,yes, -construction_work_in_progress,construction_work_in_progress,construction_work_in_progress,utility_plant_summary_ferc1,yes, -leased_to_others,utility_plant_leased_to_others,utility_plant_leased_to_others,utility_plant_summary_ferc1,yes, -asset_retirement_costs_for_distribution_plant,asset_retirement_costs_for_distribution_plant_distribution_plant,asset_retirement_costs_for_distribution_plant_distribution_plant,plant_in_service_ferc1,yes, -land_and_land_rights,land_and_land_rights_general_plant,land_and_land_rights_general_plant,plant_in_service_ferc1,yes, -structures_and_improvements,structures_and_improvements_general_plant,structures_and_improvements_general_plant,plant_in_service_ferc1,yes, -asset_retirement_costs_for_general_plant,asset_retirement_costs_for_general_plant_general_plant,asset_retirement_costs_for_general_plant_general_plant,plant_in_service_ferc1,yes, -communication_equipment,communication_equipment_general_plant,communication_equipment_general_plant,plant_in_service_ferc1,yes, -laboratory_equipment,laboratory_equipment_general_plant,laboratory_equipment_general_plant,plant_in_service_ferc1,yes, -miscellaneous_equipment,miscellaneous_equipment_general_plant,miscellaneous_equipment_general_plant,plant_in_service_ferc1,yes, -office_furniture_and_equipment,office_furniture_and_equipment_general_plant,office_furniture_and_equipment_general_plant,plant_in_service_ferc1,yes, -other_tangible_property,other_tangible_property_general_plant,other_tangible_property_general_plant,plant_in_service_ferc1,yes, -power_operated_equipment,power_operated_equipment_general_plant,power_operated_equipment_general_plant,plant_in_service_ferc1,yes, -stores_equipment,stores_equipment_general_plant,stores_equipment_general_plant,plant_in_service_ferc1,yes, -tools_shop_and_garage_equipment,tools_shop_and_garage_equipment_general_plant,tools_shop_and_garage_equipment_general_plant,plant_in_service_ferc1,yes, -transportation_equipment,transportation_equipment_general_plant,transportation_equipment_general_plant,plant_in_service_ferc1,yes, -asset_retirement_costs_for_hydraulic_production,asset_retirement_costs_for_hydraulic_production_plant_hydraulic_production,asset_retirement_costs_for_hydraulic_production_plant_hydraulic_production,plant_in_service_ferc1,yes, -franchises_and_consents,franchises_and_consents,franchises_and_consents,plant_in_service_ferc1,yes, -miscellaneous_intangible_plant,miscellaneous_intangible_plant,miscellaneous_intangible_plant,plant_in_service_ferc1,yes, -organization,organization,organization,plant_in_service_ferc1,yes, -asset_retirement_costs_for_nuclear_production,asset_retirement_costs_for_nuclear_production_plant_nuclear_production,asset_retirement_costs_for_nuclear_production_plant_nuclear_production,plant_in_service_ferc1,yes, -asset_retirement_costs_for_other_production,asset_retirement_costs_for_other_production_plant_other_production,asset_retirement_costs_for_other_production_plant_other_production,plant_in_service_ferc1,yes, -asset_retirement_costs_for_steam_production,asset_retirement_costs_for_steam_production_plant_steam_production,asset_retirement_costs_for_steam_production_plant_steam_production,plant_in_service_ferc1,yes, -asset_retirement_costs_for_transmission_plant,asset_retirement_costs_for_transmission_plant_transmission_plant,asset_retirement_costs_for_transmission_plant_transmission_plant,plant_in_service_ferc1,yes, -electric_plant_purchased,electric_plant_purchased,electric_plant_purchased,plant_in_service_ferc1,yes, -electric_plant_sold,electric_plant_sold,electric_plant_sold,plant_in_service_ferc1,yes, -utility_plant,utility_plant,utility_plant,balance_sheet_assets_ferc1,yes, -accumulated_miscellaneous_operating_provisions,accumulated_miscellaneous_operating_provisions,accumulated_miscellaneous_operating_provisions,balance_sheet_liabilities_ferc1,yes, -accumulated_provision_for_injuries_and_damages,accumulated_provision_for_injuries_and_damages,accumulated_provision_for_injuries_and_damages,balance_sheet_liabilities_ferc1,yes, -accumulated_provision_for_pensions_and_benefits,accumulated_provision_for_pensions_and_benefits,accumulated_provision_for_pensions_and_benefits,balance_sheet_liabilities_ferc1,yes, -accumulated_provision_for_property_insurance,accumulated_provision_for_property_insurance,accumulated_provision_for_property_insurance,balance_sheet_liabilities_ferc1,yes, -accumulated_provision_for_rate_refunds,accumulated_provision_for_rate_refunds,accumulated_provision_for_rate_refunds,balance_sheet_liabilities_ferc1,yes, -asset_retirement_obligations,asset_retirement_obligations,asset_retirement_obligations,balance_sheet_liabilities_ferc1,no,tend to be electric - though could be for all asset types. -depreciation_fund,depreciation_fund,depreciation_fund,balance_sheet_assets_ferc1,yes, -investment_in_subsidiary_companies,investment_in_subsidiary_companies,investment_in_subsidiary_companies,balance_sheet_assets_ferc1,yes, -nonutility_property,nonutility_property,nonutility_property,balance_sheet_assets_ferc1,no,not an asset for the good of the customers -other_investments,other_investments,other_investments,balance_sheet_assets_ferc1,no,not an asset for the good of the customers -other_special_funds,other_special_funds,other_special_funds,balance_sheet_assets_ferc1,yes,Can include securities held for decomissioning cost (VEPCo has debt & equity instruments in this account for nuclear decomissioning -sinking_funds,sinking_funds,sinking_funds,balance_sheet_assets_ferc1,yes, -special_funds,special_funds,special_funds,balance_sheet_assets_ferc1,yes, -accumulated_other_comprehensive_income,accumulated_other_comprehensive_income,accumulated_other_comprehensive_income,balance_sheet_liabilities_ferc1,no, -capital_stock_subscribed,capital_stock_subscribed,capital_stock_subscribed,balance_sheet_liabilities_ferc1,no, -common_stock_issued,common_stock_issued,common_stock_issued,balance_sheet_liabilities_ferc1,no, -installments_received_on_capital_stock,installments_received_on_capital_stock,installments_received_on_capital_stock,balance_sheet_liabilities_ferc1,no, -other_paid_in_capital,other_paid_in_capital,other_paid_in_capital,balance_sheet_liabilities_ferc1,no, -preferred_stock_issued,preferred_stock_issued,preferred_stock_issued,balance_sheet_liabilities_ferc1,no, -premium_on_capital_stock,premium_on_capital_stock,premium_on_capital_stock,balance_sheet_liabilities_ferc1,no, -appropriated_retained_earnings,appropriated_retained_earnings,appropriated_retained_earnings,retained_earnings_ferc1,no, -retained_earnings,retained_earnings,retained_earnings,balance_sheet_liabilities_ferc1,no, -retained_earnings,retained_earnings,retained_earnings,retained_earnings_ferc1,no, -unappropriated_retained_earnings,unappropriated_retained_earnings,unappropriated_retained_earnings,retained_earnings_ferc1,no, -unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,balance_sheet_liabilities_ferc1,no, -unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,no, -accounts_receivable_from_assoc_companies,accounts_receivable_from_assoc_companies,notes_receivable_from_associated_companies,balance_sheet_assets_ferc1,no,Don't believe associated companies are involved in utility services? -accum_prov_for_uncollectible_acct_credit,accum_prov_for_uncollectible_acct_credit,accumulated_provision_for_uncollectible_accounts_credit,,yes,Losses on accounts receivable which may become collectible. -cash_and_working_funds_(non_major_only),cash_and_working_funds_(non_major_only),cash_and_working_funds,,yes, -derivative_instrument_assets___hedges,derivative_instrument_assets___hedges,derivative_instrument_assets_hedges,,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -gas_stored_underground___current,gas_stored_underground___current,gas_stored_current,balance_sheet_assets_ferc1,yes, -long_term_portion_of_derivative_instrument_assets,long_term_portion_of_derivative_instrument_assets,derivative_instrument_assets_hedges_long_term,,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -long_term_portion_of_derivative_instrument_assets___hedges,long_term_portion_of_derivative_instrument_assets___hedges,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -residuals_(elec)_and_extracted_products,residuals_(elec)_and_extracted_products,residuals,balance_sheet_assets_ferc1,yes, -working_fund,working_fund,working_funds,balance_sheet_assets_ferc1,yes, -derivative_instrument_liabilities,derivative_instrument_liabilities,derivatives_instrument_liabilities,balance_sheet_liabilities_ferc1,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -derivative_instrument_liabilities___hedges,derivative_instrument_liabilities___hedges,derivative_instrument_liabilities_hedges,balance_sheet_liabilities_ferc1,yes,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? -accum_deferred_income_taxes_accel_amort,accum_deferred_income_taxes_accel_amort,accumulated_deferred_income_taxes_accelerated_amortization_property,balance_sheet_liabilities_ferc1,yes, -accum_deferred_income_taxes_other,accum_deferred_income_taxes_other,accumulated_deferred_income_taxes_other,balance_sheet_liabilities_ferc1,yes, -accum_deferred_income_taxes_other_property,accum_deferred_income_taxes_other_property,accumulated_deferred_income_taxes_other_property,balance_sheet_liabilities_ferc1,yes, -unamortized_gain_on_reaquired_debt,unamortized_gain_on_reaquired_debt,unamortized_gain_on_reacquired_debt,balance_sheet_liabilities_ferc1,no, -def_losses_from_disposition_of_utility_plt,def_losses_from_disposition_of_utility_plt,deferred_gains_from_disposition_of_utility_plant,balance_sheet_liabilities_ferc1,yes, -prelim_sur_and_invest_charges_(gas),prelim_sur_and_invest_charges_(gas),preliminary_natural_gas_survey_and_investigation_charges_and_other_preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes, -prelim_survey_and_investigation_charges_(electric),prelim_survey_and_investigation_charges_(electric),preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes, -preliminary_natural_gas_survey_and_investigation_charges,preliminary_natural_gas_survey_and_investigation_charges,other_preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes, -research_devel_and_demonstration_expend,research_devel_and_demonstration_expend,research_development_and_demonstration_expenditures,balance_sheet_assets_ferc1,no, -unamortized_debt_expenses,unamortized_debt_expenses,unamortized_debt_expense,balance_sheet_assets_ferc1,no, -unamortized_loss_on_reaquired_debt,unamortized_loss_on_reaquired_debt,unamortized_loss_on_reacquired_debt,balance_sheet_assets_ferc1,no, -gas_stored_underground___noncurrent,gas_stored_underground___noncurrent,gas_stored_underground_noncurrent,balance_sheet_assets_ferc1,yes, -(less)_reaquired_bonds,(less)_reaquired_bonds,reacquired_bonds,balance_sheet_liabilities_ferc1,no, -(less)_unamortized_discount_on_long_term_debt_debit,(less)_unamortized_discount_on_long_term_debt_debit,unamortized_discount_on_long_term_debt_debit,balance_sheet_liabilities_ferc1,no, -accum_prov_for_amort_of_nucl_fuel_assemblies,accum_prov_for_amort_of_nucl_fuel_assemblies,accumulated_provision_for_amortization_of_nuclear_fuel_assemblies,balance_sheet_assets_ferc1,yes, -nuclear_fuel,nuclear_fuel,nuclear_fuel_net,balance_sheet_assets_ferc1,yes, -nuclear_fuel_assemblies_in_reactor,nuclear_fuel_assemblies_in_reactor,nuclear_fuel_assemblies_in_reactor_major_only,balance_sheet_assets_ferc1,yes, -nuclear_fuel_in_process_of_ref_convenrich_and_fab,nuclear_fuel_in_process_of_ref_convenrich_and_fab,nuclear_fuel_in_process_of_refinement_conversion_enrichment_and_fabrication,balance_sheet_assets_ferc1,yes, -nuclear_fuel_materials_and_assemblies_stock_account,nuclear_fuel_materials_and_assemblies_stock_account,nuclear_fuel_materials_and_assemblies_stock_account_major_only,balance_sheet_assets_ferc1,yes, -spent_nuclear_fuel,spent_nuclear_fuel,spent_nuclear_fuel_major_only,balance_sheet_assets_ferc1,yes, -depreciation,depreciation,,,yes,CG: Is this depreciation_fund or depreciation_expense?? -amort_and_depl_of_producing_nat_gas,amort_and_depl_of_producing_nat_gas,amortization_and_depletion_of_producing_natural_gas_land_and_land_rights_utility_plant_in_service,utility_plant_summary_ferc1,yes, -amort_of_underground_storage,amort_of_underground_storage,amortization_of_underground_storage_land_and_land_rights_utility_plant_in_service,utility_plant_summary_ferc1,yes, -amortization_of_other_utility_plant,amortization_of_other_utility_plant,amortization_of_other_utility_plant_utility_plant_in_service,utility_plant_summary_ferc1,yes, -accum_prov_for_depr_amort_depl,accum_prov_for_depr_amort_depl,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,yes, -acquisition_adjustments,acquisition,utility_plant_acquisition_adjustment,utility_plant_summary_ferc1,yes, -held_for_future_use,held_for_future_use,utility_plant_held_for_future_use,utility_plant_summary_ferc1,yes, -completed_construction_unclassified,completed_construction_unclassified,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,yes, -experimental_plant_unclassified,experimental_plant_unclassified,experimental_electric_plant_unclassified,plant_in_service_ferc1,yes, -plant_in_service,plant_in_service,electric_plant_in_service,plant_in_service_ferc1,yes, -installations_on_customer_premises,installations_on_customer_premises,installations_on_customer_premises_distribution_plant,plant_in_service_ferc1,yes, -leased_property_on_customer_premises,leased_property_on_customer_premises,leased_property_on_customer_premises_distribution_plant,plant_in_service_ferc1,yes, -line_transformers,line_transformers,line_transformers_distribution_plant,plant_in_service_ferc1,yes,CG: Is this line_transformers_distribution_plant/plant_in_service_ferc1 or maintenance_of_line_transformers/electric_operating_expenses_ferc1? -meters,meters,meters_distribution_plant,plant_in_service_ferc1,yes,CG: Is this meters_distribution_plant/plant_in_service_ferc1 or maintenance_of_meters/electric_operating_expenses_ferc1? -overhead_conductors_and_devices,overhead_conductors_and_devices,overhead_conductors_and_devices_transmission_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -overhead_conductors_and_devices,overhead_conductors_and_devices,overhead_conductors_and_devices_distribution_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -poles_towers_and_fixtures,poles_towers_and_fixtures,poles_towers_and_fixtures_distribution_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -services,services,services_distribution_plant,plant_in_service_ferc1,yes, -station_equipment,station_equipment,station_equipment_transmission_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -station_equipment,station_equipment,station_equipment_distribution_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -storage_battery_equipment,storage_battery_equipment,energy_storage_equipment_other_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -storage_battery_equipment,storage_battery_equipment,energy_storage_equipment_transmission_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -storage_battery_equipment,storage_battery_equipment,energy_storage_equipment_distribution_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -street_lighting_and_signal_systems,street_lighting_and_signal_systems,street_lighting_and_signal_systems_distribution_plant,plant_in_service_ferc1,yes, -underground_conductors_and_devices,underground_conductors_and_devices,underground_conductors_and_devices_transmission_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -underground_conductors_and_devices,underground_conductors_and_devices,underground_conductors_and_devices_distribution_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -underground_conduit,underground_conduit,underground_conduit_transmission_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -underground_conduit,underground_conduit,underground_conduit_distribution_plant,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_steam_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_nuclear_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_hydraulic_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_other_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_steam_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_nuclear_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_hydraulic_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_other_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -reservoirs_dams_and_waterways,reservoirs_dams_and_waterways,reservoirs_dams_and_waterways_hydraulic_production,plant_in_service_ferc1,yes, -roads_railroads_and_bridges,roads_railroads_and_bridges,roads_railroads_and_bridges_hydraulic_production,plant_in_service_ferc1,yes, -water_wheels_turbines_and_generators,water_wheels_turbines_and_generators,water_wheels_turbines_and_generators_hydraulic_production,plant_in_service_ferc1,yes, -reactor_plant_equipment,reactor_plant_equipment,reactor_plant_equipment_nuclear_production,plant_in_service_ferc1,yes, -turbogenerator_units,turbogenerator_units,turbogenerator_units_steam_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -turbogenerator_units,turbogenerator_units,turbogenerator_units_nuclear_production,plant_in_service_ferc1,yes,CG: We have transmission and distribution lines… should they be broken out into two records? -fuel_holders_products_and_accessories,fuel_holders_products_and_accessories,fuel_holders_products_and_accessories_other_production,plant_in_service_ferc1,yes, -generators,generators,generators_other_production,plant_in_service_ferc1,yes, -prime_movers,prime_movers,prime_movers_other_production,plant_in_service_ferc1,yes, -computer_hardware,computer_hardware,computer_hardware_regional_transmission_and_market_operation_plant,plant_in_service_ferc1,yes, -computer_software,computer_software,computer_software_regional_transmission_and_market_operation_plant,plant_in_service_ferc1,yes, -miscellaneous_regional_transmission_and_market_operation_plant,miscellaneous_regional_transmission_and_market_operation_plant,miscellaneous_regional_transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,plant_in_service_ferc1,yes, -boiler_plant_equipment,boiler_plant_equipment,boiler_plant_equipment_steam_production,plant_in_service_ferc1,yes, -engines_and_engine-driven_generators,engines_and_engine-driven_generators,engines_and_engine_driven_generators_steam_production,plant_in_service_ferc1,yes, -poles_and_fixtures,poles_and_fixtures,poles_and_fixtures_transmission_plant,plant_in_service_ferc1,yes, -roads_and_trails,roads_and_trails,roads_and_trails_transmission_plant,plant_in_service_ferc1,yes, -towers_and_fixtures,towers_and_fixtures,towers_and_fixtures_transmission_plant,plant_in_service_ferc1,yes, -plant_purchased_or_sold,plant_purchased_or_sold,utility_plant_in_service_plant_purchased_or_sold,utility_plant_summary_ferc1,yes, -property_under_capital_leases,property_under_capital_leases,utility_plant_in_service_property_under_capital_leases,utility_plant_summary_ferc1,yes, -long_term_portion_of_derivative_instrument_liabilities___hedges,long_term_portion_of_derivative_instrument_liabilities___hedges,long_term_portion_of_derivative_instrument_liabilities_hedges,balance_sheet_liabilities_ferc1,yes,Ryan/David spoke - believe derivatives & hedges use customer funds to provide benefit - just like physical assets. -obligations_under_capital_leases___noncurrent,obligations_under_capital_leases___noncurrent,obligations_under_capital_lease_noncurrent,balance_sheet_liabilities_ferc1,no,"Cost of lease would be expensed, book value would not be in rate base." -accum_prov_for_depr_and_amort,accum_prov_for_depr_and_amort,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,balance_sheet_liabilities_ferc1,yes,"4/13/21 - Goes with ""nonutility_property"" and should match on inclusion or exclusion from rate base." -accum_prov_for_depr_and_amort,accum_prov_for_depr_and_amort,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,yes,"CG: This factoid shows up in two separate tables, so this line was duplicated." -amortization_fund___federal,amortization_fund___federal,amortization_fund_federal,balance_sheet_assets_ferc1,yes, -investments_in_associated_companies,investments_in_associated_companies,investment_in_associated_companies,balance_sheet_assets_ferc1,yes, -long_term_portion_of_derivative_assets,long_term_portion_of_derivative_assets,derivative_instrument_assets_long_term,balance_sheet_assets_ferc1,yes,Ryan/David spoke - believe derivatives & hedges use customer funds to provide benefit - just like physical assets. -long_term_portion_of_derivative_assets_–_hedges,long_term_portion_of_derivative_assets_–_hedges,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,Ryan/David spoke - believe derivatives & hedges use customer funds to provide benefit - just like physical assets. -special_funds_(non_major_only),special_funds_(non_major_only),special_funds_all,balance_sheet_assets_ferc1,yes, -special_funds_(non_major_only),special_funds_(non_major_only),other_special_funds,balance_sheet_assets_ferc1,yes, -special_funds_(non_major_only),special_funds_(non_major_only),special_funds,balance_sheet_assets_ferc1,yes, -special_funds_(non_major_only),special_funds_(non_major_only),special_deposits,balance_sheet_assets_ferc1,yes, -(less)_capital_stock_expense,(less)_capital_stock_expense,capital_stock_expense,balance_sheet_liabilities_ferc1,no, -(less)_discount_on_capital_stock,(less)_discount_on_capital_stock,discount_on_capital_stock,balance_sheet_liabilities_ferc1,no, -(less)_reaquired_capital_stock,(less)_reaquired_capital_stock,reacquired_capital_stock,balance_sheet_liabilities_ferc1,no, -_noncorporate_proprietorship_(non_major_only),_noncorporate_proprietorship_(non_major_only),noncorporate_proprietorship,balance_sheet_liabilities_ferc1,no, -appropriated_retained_earnings_amort_reserve_federal,appropriated_retained_earnings_amort_reserve_federal,appropriated_retained_earnings_amortization_reserve_federal,retained_earnings_ferc1,no, +xbrl_factoid_rmi,xbrl_factoid_rmi_renamed,xbrl_factoid,table_name,in_rate_base,utility_type,plant_function,plant_status,notes +accrued_utility_revenues,accrued_utility_revenues,accrued_utility_revenues,balance_sheet_assets_ferc1,yes,,,,"At the option of the utility, the estimated amount accrued for service rendered, but not billed at the end of any accounting period, may be included herein. In case accruals are made for unbilled revenues, they shall be made likewise for unbilled expenses, such as for the purchase of energy." +advances_for_gas,advances_for_gas,advances_for_gas,balance_sheet_assets_ferc1,yes,,,,"4/13/21 - changed from ""non-electric"" to ""mixed"" - could be for gas power plants." +allowances,allowances,allowances,electric_operating_expenses_ferc1,yes,,,, +cash,cash,cash,balance_sheet_assets_ferc1,yes,,,, +customer_accounts_receivable,customer_accounts_receivable,customer_accounts_receivable,balance_sheet_assets_ferc1,yes,,,,"service provided & billed/collected, asset earned on." +derivative_instrument_assets,derivative_instrument_assets,derivative_instrument_assets,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +fuel_stock,fuel_stock,fuel_stock,balance_sheet_assets_ferc1,yes,,,, +fuel_stock_expenses_undistributed,fuel_stock_expenses_undistributed,fuel_stock_expenses_undistributed,balance_sheet_assets_ferc1,yes,,,, +interest_and_dividends_receivable,interest_and_dividends_receivable,interest_and_dividends_receivable,balance_sheet_assets_ferc1,no,,,,not customer deposits or cash working capital +liquefied_natural_gas_stored_and_held_for_processing,liquefied_natural_gas_stored_and_held_for_processing,liquefied_natural_gas_stored_and_held_for_processing,balance_sheet_assets_ferc1,yes,,,, +merchandise,merchandise,merchandise,balance_sheet_assets_ferc1,yes,,,, +miscellaneous_current_and_accrued_assets,miscellaneous_current_and_accrued_assets,miscellaneous_current_and_accrued_assets,balance_sheet_assets_ferc1,yes,,,,includes customer credits/refunds if ordered by regulatory body +noncurrent_portion_of_allowances,noncurrent_portion_of_allowances,noncurrent_portion_of_allowances,balance_sheet_assets_ferc1,yes,,,, +notes_receivable,notes_receivable,notes_receivable,balance_sheet_assets_ferc1,yes,,,, +notes_receivable_from_associated_companies,notes_receivable_from_associated_companies,notes_receivable_from_associated_companies,balance_sheet_assets_ferc1,no,,,,Don't believe associated companies are involved in utility services? +nuclear_materials_held_for_sale,nuclear_materials_held_for_sale,nuclear_materials_held_for_sale,balance_sheet_assets_ferc1,yes,,,, +other_accounts_receivable,other_accounts_receivable,other_accounts_receivable,balance_sheet_assets_ferc1,yes,,,, +other_materials_and_supplies,other_materials_and_supplies,other_materials_and_supplies,balance_sheet_assets_ferc1,yes,,,, +plant_materials_and_operating_supplies,plant_materials_and_operating_supplies,plant_materials_and_operating_supplies,balance_sheet_assets_ferc1,yes,,,, +prepayments,prepayments,prepayments,balance_sheet_assets_ferc1,yes,,,, +rents_receivable,rents_receivable,rents_receivable,balance_sheet_assets_ferc1,yes,,,,asset of customers/ratepayers +special_deposits,special_deposits,special_deposits,balance_sheet_assets_ferc1,yes,,,,customer deposits - restricted cash +stores_expense_undistributed,stores_expense_undistributed,stores_expense_undistributed,balance_sheet_assets_ferc1,yes,,,, +temporary_cash_investments,temporary_cash_investments,temporary_cash_investments,balance_sheet_assets_ferc1,yes,,,, +(less)_long_term_portion_of_derivative_instrument_liabilities,long_term_portion_of_derivative_instrument_liabilities,long_term_portion_of_derivative_instrument_liabilities,balance_sheet_liabilities_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +(less)_long_term_portion_of_derivative_instrument_liabilities_hedges,long_term_portion_of_derivative_instrument_liabilities_hedges,long_term_portion_of_derivative_instrument_liabilities_hedges,balance_sheet_liabilities_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +accounts_payable,accounts_payable,accounts_payable,balance_sheet_liabilities_ferc1,yes,,,, +accounts_payable_to_associated_companies,accounts_payable_to_associated_companies,accounts_payable_to_associated_companies,balance_sheet_liabilities_ferc1,no,,,,Don't believe associated companies are involved in utility services? +customer_deposits,customer_deposits,customer_deposits,balance_sheet_liabilities_ferc1,yes,,,, +dividends_declared,dividends_declared,dividends_declared,balance_sheet_liabilities_ferc1,no,,,, +interest_accrued,interest_accrued,interest_accrued,balance_sheet_liabilities_ferc1,no,,,, +matured_interest,matured_interest,matured_interest,balance_sheet_liabilities_ferc1,no,,,,not cash working capital +matured_long_term_debt,matured_long_term_debt,matured_long_term_debt,balance_sheet_liabilities_ferc1,no,,,,not cash working capital +miscellaneous_current_and_accrued_liabilities,miscellaneous_current_and_accrued_liabilities,miscellaneous_current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,yes,,,,includes customer credits/refunds if ordered by regulatory body +notes_payable,notes_payable,notes_payable,balance_sheet_liabilities_ferc1,yes,,,, +notes_payable_to_associated_companies,notes_payable_to_associated_companies,notes_payable_to_associated_companies,balance_sheet_liabilities_ferc1,yes,,,, +obligations_under_capital_leases_current,obligations_under_capital_leases_current,obligations_under_capital_leases_current,balance_sheet_liabilities_ferc1,yes,,,, +tax_collections_payable,tax_collections_payable,tax_collections_payable,balance_sheet_liabilities_ferc1,yes,,,, +taxes_accrued,taxes_accrued,taxes_accrued,balance_sheet_liabilities_ferc1,yes,,,, +accumulated_deferred_income_taxes,accumulated_deferred_income_taxes,accumulated_deferred_income_taxes,balance_sheet_assets_ferc1,yes,,,, +accumulated_deferred_investment_tax_credits,accumulated_deferred_investment_tax_credits,accumulated_deferred_investment_tax_credits,balance_sheet_liabilities_ferc1,yes,,,, +customer_advances_for_construction,customer_advances_for_construction,customer_advances_for_construction,balance_sheet_liabilities_ferc1,yes,,,, +deferred_gains_from_disposition_of_utility_plant,deferred_gains_from_disposition_of_utility_plant,deferred_gains_from_disposition_of_utility_plant,balance_sheet_liabilities_ferc1,yes,,,, +other_deferred_credits,other_deferred_credits,other_deferred_credits,balance_sheet_liabilities_ferc1,yes,,,,"Includes things such as ""PJM Transmission Cost Allocation"", ""Spent Nuclear Fuel - Other"", and similar items." +other_regulatory_liabilities,other_regulatory_liabilities,other_regulatory_liabilities,balance_sheet_liabilities_ferc1,yes,,,, +clearing_accounts,clearing_accounts,clearing_accounts,balance_sheet_assets_ferc1,yes,,,, +extraordinary_property_losses,extraordinary_property_losses,extraordinary_property_losses,balance_sheet_assets_ferc1,no,,,, +miscellaneous_deferred_debits,miscellaneous_deferred_debits,miscellaneous_deferred_debits,balance_sheet_assets_ferc1,yes,,,, +other_preliminary_survey_and_investigation_charges,other_preliminary_survey_and_investigation_charges,other_preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes,,,, +other_regulatory_assets,other_regulatory_assets,other_regulatory_assets,balance_sheet_assets_ferc1,yes,,,, +temporary_facilities,temporary_facilities,temporary_facilities,balance_sheet_assets_ferc1,no,,,, +unrecovered_plant_and_regulatory_study_costs,unrecovered_plant_and_regulatory_study_costs,unrecovered_plant_and_regulatory_study_costs,balance_sheet_assets_ferc1,yes,,,, +unrecovered_purchased_gas_costs,unrecovered_purchased_gas_costs,unrecovered_purchased_gas_costs,balance_sheet_assets_ferc1,yes,,,, +advances_from_associated_companies,advances_from_associated_companies,advances_from_associated_companies,balance_sheet_liabilities_ferc1,no,,,,Don't believe associated companies are involved in utility services? +bonds,bonds,bonds,balance_sheet_liabilities_ferc1,no,,,, +other_long_term_debt,other_long_term_debt,other_long_term_debt,balance_sheet_liabilities_ferc1,no,,,, +unamortized_premium_on_long_term_debt,unamortized_premium_on_long_term_debt,unamortized_premium_on_long_term_debt,balance_sheet_liabilities_ferc1,no,,,, +nuclear_fuel_under_capital_leases,nuclear_fuel_under_capital_leases,nuclear_fuel_under_capital_leases,balance_sheet_assets_ferc1,yes,,,, +abandonment_of_leases,abandonment_of_leases,abandonment_of_leases,utility_plant_summary_ferc1,yes,,,, +amortization_of_plant_acquisition_adjustment,amortization_of_plant_acquisition_adjustment,amortization_of_plant_acquisition_adjustment,utility_plant_summary_ferc1,yes,,,, +amortization,amortization_utility_plant_held_for_future_use,amortization_utility_plant_held_for_future_use,utility_plant_summary_ferc1,yes,,,, +construction_work_in_progress,construction_work_in_progress,construction_work_in_progress,balance_sheet_assets_ferc1,yes,,,, +construction_work_in_progress,construction_work_in_progress,construction_work_in_progress,utility_plant_summary_ferc1,yes,,,, +leased_to_others,utility_plant_leased_to_others,utility_plant_leased_to_others,utility_plant_summary_ferc1,yes,,,, +asset_retirement_costs_for_distribution_plant,asset_retirement_costs_for_distribution_plant_distribution_plant,asset_retirement_costs_for_distribution_plant_distribution_plant,plant_in_service_ferc1,yes,,,, +land_and_land_rights,land_and_land_rights_general_plant,land_and_land_rights_general_plant,plant_in_service_ferc1,yes,,,, +structures_and_improvements,structures_and_improvements_general_plant,structures_and_improvements_general_plant,plant_in_service_ferc1,yes,,,, +asset_retirement_costs_for_general_plant,asset_retirement_costs_for_general_plant_general_plant,asset_retirement_costs_for_general_plant_general_plant,plant_in_service_ferc1,yes,,,, +communication_equipment,communication_equipment_general_plant,communication_equipment_general_plant,plant_in_service_ferc1,yes,,,, +laboratory_equipment,laboratory_equipment_general_plant,laboratory_equipment_general_plant,plant_in_service_ferc1,yes,,,, +miscellaneous_equipment,miscellaneous_equipment_general_plant,miscellaneous_equipment_general_plant,plant_in_service_ferc1,yes,,,, +office_furniture_and_equipment,office_furniture_and_equipment_general_plant,office_furniture_and_equipment_general_plant,plant_in_service_ferc1,yes,,,, +other_tangible_property,other_tangible_property_general_plant,other_tangible_property_general_plant,plant_in_service_ferc1,yes,,,, +power_operated_equipment,power_operated_equipment_general_plant,power_operated_equipment_general_plant,plant_in_service_ferc1,yes,,,, +stores_equipment,stores_equipment_general_plant,stores_equipment_general_plant,plant_in_service_ferc1,yes,,,, +tools_shop_and_garage_equipment,tools_shop_and_garage_equipment_general_plant,tools_shop_and_garage_equipment_general_plant,plant_in_service_ferc1,yes,,,, +transportation_equipment,transportation_equipment_general_plant,transportation_equipment_general_plant,plant_in_service_ferc1,yes,,,, +asset_retirement_costs_for_hydraulic_production,asset_retirement_costs_for_hydraulic_production_plant_hydraulic_production,asset_retirement_costs_for_hydraulic_production_plant_hydraulic_production,plant_in_service_ferc1,yes,,,, +franchises_and_consents,franchises_and_consents,franchises_and_consents,plant_in_service_ferc1,yes,,,, +miscellaneous_intangible_plant,miscellaneous_intangible_plant,miscellaneous_intangible_plant,plant_in_service_ferc1,yes,,,, +organization,organization,organization,plant_in_service_ferc1,yes,,,, +asset_retirement_costs_for_nuclear_production,asset_retirement_costs_for_nuclear_production_plant_nuclear_production,asset_retirement_costs_for_nuclear_production_plant_nuclear_production,plant_in_service_ferc1,yes,,,, +asset_retirement_costs_for_other_production,asset_retirement_costs_for_other_production_plant_other_production,asset_retirement_costs_for_other_production_plant_other_production,plant_in_service_ferc1,yes,,,, +asset_retirement_costs_for_steam_production,asset_retirement_costs_for_steam_production_plant_steam_production,asset_retirement_costs_for_steam_production_plant_steam_production,plant_in_service_ferc1,yes,,,, +asset_retirement_costs_for_transmission_plant,asset_retirement_costs_for_transmission_plant_transmission_plant,asset_retirement_costs_for_transmission_plant_transmission_plant,plant_in_service_ferc1,yes,,,, +electric_plant_purchased,electric_plant_purchased,electric_plant_purchased,plant_in_service_ferc1,yes,,,, +electric_plant_sold,electric_plant_sold,electric_plant_sold,plant_in_service_ferc1,yes,,,, +utility_plant,utility_plant,utility_plant,balance_sheet_assets_ferc1,yes,,,, +accumulated_miscellaneous_operating_provisions,accumulated_miscellaneous_operating_provisions,accumulated_miscellaneous_operating_provisions,balance_sheet_liabilities_ferc1,yes,,,, +accumulated_provision_for_injuries_and_damages,accumulated_provision_for_injuries_and_damages,accumulated_provision_for_injuries_and_damages,balance_sheet_liabilities_ferc1,yes,,,, +accumulated_provision_for_pensions_and_benefits,accumulated_provision_for_pensions_and_benefits,accumulated_provision_for_pensions_and_benefits,balance_sheet_liabilities_ferc1,yes,,,, +accumulated_provision_for_property_insurance,accumulated_provision_for_property_insurance,accumulated_provision_for_property_insurance,balance_sheet_liabilities_ferc1,yes,,,, +accumulated_provision_for_rate_refunds,accumulated_provision_for_rate_refunds,accumulated_provision_for_rate_refunds,balance_sheet_liabilities_ferc1,yes,,,, +asset_retirement_obligations,asset_retirement_obligations,asset_retirement_obligations,balance_sheet_liabilities_ferc1,no,,,,tend to be electric - though could be for all asset types. +depreciation_fund,depreciation_fund,depreciation_fund,balance_sheet_assets_ferc1,yes,,,, +investment_in_subsidiary_companies,investment_in_subsidiary_companies,investment_in_subsidiary_companies,balance_sheet_assets_ferc1,yes,,,, +nonutility_property,nonutility_property,nonutility_property,balance_sheet_assets_ferc1,no,,,,not an asset for the good of the customers +other_investments,other_investments,other_investments,balance_sheet_assets_ferc1,no,,,,not an asset for the good of the customers +other_special_funds,other_special_funds,other_special_funds,balance_sheet_assets_ferc1,yes,,,,Can include securities held for decomissioning cost (VEPCo has debt & equity instruments in this account for nuclear decomissioning +sinking_funds,sinking_funds,sinking_funds,balance_sheet_assets_ferc1,yes,,,, +special_funds,special_funds,special_funds,balance_sheet_assets_ferc1,yes,,,, +accumulated_other_comprehensive_income,accumulated_other_comprehensive_income,accumulated_other_comprehensive_income,balance_sheet_liabilities_ferc1,no,,,, +capital_stock_subscribed,capital_stock_subscribed,capital_stock_subscribed,balance_sheet_liabilities_ferc1,no,,,, +common_stock_issued,common_stock_issued,common_stock_issued,balance_sheet_liabilities_ferc1,no,,,, +installments_received_on_capital_stock,installments_received_on_capital_stock,installments_received_on_capital_stock,balance_sheet_liabilities_ferc1,no,,,, +other_paid_in_capital,other_paid_in_capital,other_paid_in_capital,balance_sheet_liabilities_ferc1,no,,,, +preferred_stock_issued,preferred_stock_issued,preferred_stock_issued,balance_sheet_liabilities_ferc1,no,,,, +premium_on_capital_stock,premium_on_capital_stock,premium_on_capital_stock,balance_sheet_liabilities_ferc1,no,,,, +appropriated_retained_earnings,appropriated_retained_earnings,appropriated_retained_earnings,retained_earnings_ferc1,no,,,, +retained_earnings,retained_earnings,retained_earnings,balance_sheet_liabilities_ferc1,no,,,, +retained_earnings,retained_earnings,retained_earnings,retained_earnings_ferc1,no,,,, +unappropriated_retained_earnings,unappropriated_retained_earnings,unappropriated_retained_earnings,retained_earnings_ferc1,no,,,, +unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,balance_sheet_liabilities_ferc1,no,,,, +unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,no,,,, +accounts_receivable_from_assoc_companies,accounts_receivable_from_assoc_companies,notes_receivable_from_associated_companies,balance_sheet_assets_ferc1,no,,,,Don't believe associated companies are involved in utility services? +accum_prov_for_uncollectible_acct_credit,accum_prov_for_uncollectible_acct_credit,accumulated_provision_for_uncollectible_accounts_credit,,yes,,,,Losses on accounts receivable which may become collectible. +cash_and_working_funds_(non_major_only),cash_and_working_funds_(non_major_only),cash_and_working_funds,,yes,,,, +derivative_instrument_assets___hedges,derivative_instrument_assets___hedges,derivative_instrument_assets_hedges,,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +gas_stored_underground___current,gas_stored_underground___current,gas_stored_current,balance_sheet_assets_ferc1,yes,,,, +long_term_portion_of_derivative_instrument_assets,long_term_portion_of_derivative_instrument_assets,derivative_instrument_assets_hedges_long_term,,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +long_term_portion_of_derivative_instrument_assets___hedges,long_term_portion_of_derivative_instrument_assets___hedges,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +residuals_(elec)_and_extracted_products,residuals_(elec)_and_extracted_products,residuals,balance_sheet_assets_ferc1,yes,,,, +working_fund,working_fund,working_funds,balance_sheet_assets_ferc1,yes,,,, +derivative_instrument_liabilities,derivative_instrument_liabilities,derivatives_instrument_liabilities,balance_sheet_liabilities_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +derivative_instrument_liabilities___hedges,derivative_instrument_liabilities___hedges,derivative_instrument_liabilities_hedges,balance_sheet_liabilities_ferc1,yes,,,,Ryan/David spoke - believe hedges to be part of rate base - capital put up by customers for their benefit? +accum_deferred_income_taxes_accel_amort,accum_deferred_income_taxes_accel_amort,accumulated_deferred_income_taxes_accelerated_amortization_property,balance_sheet_liabilities_ferc1,yes,,,, +accum_deferred_income_taxes_other,accum_deferred_income_taxes_other,accumulated_deferred_income_taxes_other,balance_sheet_liabilities_ferc1,yes,,,, +accum_deferred_income_taxes_other_property,accum_deferred_income_taxes_other_property,accumulated_deferred_income_taxes_other_property,balance_sheet_liabilities_ferc1,yes,,,, +unamortized_gain_on_reaquired_debt,unamortized_gain_on_reaquired_debt,unamortized_gain_on_reacquired_debt,balance_sheet_liabilities_ferc1,no,,,, +def_losses_from_disposition_of_utility_plt,def_losses_from_disposition_of_utility_plt,deferred_gains_from_disposition_of_utility_plant,balance_sheet_liabilities_ferc1,yes,,,, +prelim_sur_and_invest_charges_(gas),prelim_sur_and_invest_charges_(gas),preliminary_natural_gas_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes,,,, +prelim_survey_and_investigation_charges_(electric),prelim_survey_and_investigation_charges_(electric),preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes,,,, +preliminary_natural_gas_survey_and_investigation_charges,preliminary_natural_gas_survey_and_investigation_charges,other_preliminary_survey_and_investigation_charges,balance_sheet_assets_ferc1,yes,,,, +research_devel_and_demonstration_expend,research_devel_and_demonstration_expend,research_development_and_demonstration_expenditures,balance_sheet_assets_ferc1,no,,,, +unamortized_debt_expenses,unamortized_debt_expenses,unamortized_debt_expense,balance_sheet_assets_ferc1,no,,,, +unamortized_loss_on_reaquired_debt,unamortized_loss_on_reaquired_debt,unamortized_loss_on_reacquired_debt,balance_sheet_assets_ferc1,no,,,, +gas_stored_underground___noncurrent,gas_stored_underground___noncurrent,gas_stored_underground_noncurrent,balance_sheet_assets_ferc1,yes,,,, +(less)_reaquired_bonds,(less)_reaquired_bonds,reacquired_bonds,balance_sheet_liabilities_ferc1,no,,,, +(less)_unamortized_discount_on_long_term_debt_debit,(less)_unamortized_discount_on_long_term_debt_debit,unamortized_discount_on_long_term_debt_debit,balance_sheet_liabilities_ferc1,no,,,, +accum_prov_for_amort_of_nucl_fuel_assemblies,accum_prov_for_amort_of_nucl_fuel_assemblies,accumulated_provision_for_amortization_of_nuclear_fuel_assemblies,balance_sheet_assets_ferc1,yes,,,, +nuclear_fuel,nuclear_fuel,nuclear_fuel_net,balance_sheet_assets_ferc1,yes,,,, +nuclear_fuel_assemblies_in_reactor,nuclear_fuel_assemblies_in_reactor,nuclear_fuel_assemblies_in_reactor_major_only,balance_sheet_assets_ferc1,yes,,,, +nuclear_fuel_in_process_of_ref_convenrich_and_fab,nuclear_fuel_in_process_of_ref_convenrich_and_fab,nuclear_fuel_in_process_of_refinement_conversion_enrichment_and_fabrication,balance_sheet_assets_ferc1,yes,,,, +nuclear_fuel_materials_and_assemblies_stock_account,nuclear_fuel_materials_and_assemblies_stock_account,nuclear_fuel_materials_and_assemblies_stock_account_major_only,balance_sheet_assets_ferc1,yes,,,, +spent_nuclear_fuel,spent_nuclear_fuel,spent_nuclear_fuel_major_only,balance_sheet_assets_ferc1,yes,,,, +amort_and_depl_of_producing_nat_gas,amort_and_depl_of_producing_nat_gas,amortization_and_depletion_of_producing_natural_gas_land_and_land_rights_utility_plant_in_service,utility_plant_summary_ferc1,yes,,,, +amort_of_underground_storage,amort_of_underground_storage,amortization_of_underground_storage_land_and_land_rights_utility_plant_in_service,utility_plant_summary_ferc1,yes,,,, +amortization_of_other_utility_plant,amortization_of_other_utility_plant,amortization_of_other_utility_plant_utility_plant_in_service,utility_plant_summary_ferc1,yes,,,, +accum_prov_for_depr_amort_depl,accum_prov_for_depr_amort_depl,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,yes,,,, +acquisition_adjustments,acquisition,utility_plant_acquisition_adjustment,utility_plant_summary_ferc1,yes,,,, +held_for_future_use,held_for_future_use,utility_plant_held_for_future_use,utility_plant_summary_ferc1,yes,,,, +completed_construction_unclassified,completed_construction_unclassified,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,yes,,,, +experimental_plant_unclassified,experimental_plant_unclassified,experimental_electric_plant_unclassified,plant_in_service_ferc1,yes,,,, +plant_in_service,plant_in_service,electric_plant_in_service,plant_in_service_ferc1,yes,,,, +installations_on_customer_premises,installations_on_customer_premises,installations_on_customer_premises_distribution_plant,plant_in_service_ferc1,yes,,,, +leased_property_on_customer_premises,leased_property_on_customer_premises,leased_property_on_customer_premises_distribution_plant,plant_in_service_ferc1,yes,,,, +line_transformers,line_transformers,line_transformers_distribution_plant,plant_in_service_ferc1,yes,,,,CG: Is this line_transformers_distribution_plant/plant_in_service_ferc1 or maintenance_of_line_transformers/electric_operating_expenses_ferc1? +meters,meters,meters_distribution_plant,plant_in_service_ferc1,yes,,,,CG: Is this meters_distribution_plant/plant_in_service_ferc1 or maintenance_of_meters/electric_operating_expenses_ferc1? +overhead_conductors_and_devices,overhead_conductors_and_devices,overhead_conductors_and_devices_transmission_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +overhead_conductors_and_devices,overhead_conductors_and_devices,overhead_conductors_and_devices_distribution_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +poles_towers_and_fixtures,poles_towers_and_fixtures,poles_towers_and_fixtures_distribution_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +services,services,services_distribution_plant,plant_in_service_ferc1,yes,,,, +station_equipment,station_equipment,station_equipment_transmission_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +station_equipment,station_equipment,station_equipment_distribution_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +storage_battery_equipment,storage_battery_equipment,energy_storage_equipment_other_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +storage_battery_equipment,storage_battery_equipment,energy_storage_equipment_transmission_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +storage_battery_equipment,storage_battery_equipment,energy_storage_equipment_distribution_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +street_lighting_and_signal_systems,street_lighting_and_signal_systems,street_lighting_and_signal_systems_distribution_plant,plant_in_service_ferc1,yes,,,, +underground_conductors_and_devices,underground_conductors_and_devices,underground_conductors_and_devices_transmission_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +underground_conductors_and_devices,underground_conductors_and_devices,underground_conductors_and_devices_distribution_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +underground_conduit,underground_conduit,underground_conduit_transmission_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +underground_conduit,underground_conduit,underground_conduit_distribution_plant,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_steam_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_nuclear_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_hydraulic_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +accessory_electric_equipment,accessory_electric_equipment,accessory_electric_equipment_other_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_steam_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_nuclear_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_hydraulic_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +misc._power_plant_equipment,misc._power_plant_equipment,miscellaneous_power_plant_equipment_other_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +reservoirs_dams_and_waterways,reservoirs_dams_and_waterways,reservoirs_dams_and_waterways_hydraulic_production,plant_in_service_ferc1,yes,,,, +roads_railroads_and_bridges,roads_railroads_and_bridges,roads_railroads_and_bridges_hydraulic_production,plant_in_service_ferc1,yes,,,, +water_wheels_turbines_and_generators,water_wheels_turbines_and_generators,water_wheels_turbines_and_generators_hydraulic_production,plant_in_service_ferc1,yes,,,, +reactor_plant_equipment,reactor_plant_equipment,reactor_plant_equipment_nuclear_production,plant_in_service_ferc1,yes,,,, +turbogenerator_units,turbogenerator_units,turbogenerator_units_steam_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +turbogenerator_units,turbogenerator_units,turbogenerator_units_nuclear_production,plant_in_service_ferc1,yes,,,,CG: We have transmission and distribution lines… should they be broken out into two records? +fuel_holders_products_and_accessories,fuel_holders_products_and_accessories,fuel_holders_products_and_accessories_other_production,plant_in_service_ferc1,yes,,,, +generators,generators,generators_other_production,plant_in_service_ferc1,yes,,,, +prime_movers,prime_movers,prime_movers_other_production,plant_in_service_ferc1,yes,,,, +computer_hardware,computer_hardware,computer_hardware_regional_transmission_and_market_operation_plant,plant_in_service_ferc1,yes,,,, +computer_software,computer_software,computer_software_regional_transmission_and_market_operation_plant,plant_in_service_ferc1,yes,,,, +miscellaneous_regional_transmission_and_market_operation_plant,miscellaneous_regional_transmission_and_market_operation_plant,miscellaneous_regional_transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,plant_in_service_ferc1,yes,,,, +boiler_plant_equipment,boiler_plant_equipment,boiler_plant_equipment_steam_production,plant_in_service_ferc1,yes,,,, +engines_and_engine-driven_generators,engines_and_engine-driven_generators,engines_and_engine_driven_generators_steam_production,plant_in_service_ferc1,yes,,,, +poles_and_fixtures,poles_and_fixtures,poles_and_fixtures_transmission_plant,plant_in_service_ferc1,yes,,,, +roads_and_trails,roads_and_trails,roads_and_trails_transmission_plant,plant_in_service_ferc1,yes,,,, +towers_and_fixtures,towers_and_fixtures,towers_and_fixtures_transmission_plant,plant_in_service_ferc1,yes,,,, +plant_purchased_or_sold,plant_purchased_or_sold,utility_plant_in_service_plant_purchased_or_sold,utility_plant_summary_ferc1,yes,,,, +property_under_capital_leases,property_under_capital_leases,utility_plant_in_service_property_under_capital_leases,utility_plant_summary_ferc1,yes,,,, +long_term_portion_of_derivative_instrument_liabilities___hedges,long_term_portion_of_derivative_instrument_liabilities___hedges,long_term_portion_of_derivative_instrument_liabilities_hedges,balance_sheet_liabilities_ferc1,yes,,,,Ryan/David spoke - believe derivatives & hedges use customer funds to provide benefit - just like physical assets. +obligations_under_capital_leases___noncurrent,obligations_under_capital_leases___noncurrent,obligations_under_capital_lease_noncurrent,balance_sheet_liabilities_ferc1,no,,,,"Cost of lease would be expensed, book value would not be in rate base." +accum_prov_for_depr_and_amort,accum_prov_for_depr_and_amort,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,balance_sheet_liabilities_ferc1,yes,,,,"4/13/21 - Goes with ""nonutility_property"" and should match on inclusion or exclusion from rate base." +accum_prov_for_depr_and_amort,accum_prov_for_depr_and_amort,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,yes,,,,"CG: This factoid shows up in two separate tables, so this line was duplicated." +amortization_fund___federal,amortization_fund___federal,amortization_fund_federal,balance_sheet_assets_ferc1,yes,,,, +investments_in_associated_companies,investments_in_associated_companies,investment_in_associated_companies,balance_sheet_assets_ferc1,yes,,,, +long_term_portion_of_derivative_assets,long_term_portion_of_derivative_assets,derivative_instrument_assets_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe derivatives & hedges use customer funds to provide benefit - just like physical assets. +long_term_portion_of_derivative_assets_–_hedges,long_term_portion_of_derivative_assets_–_hedges,derivative_instrument_assets_hedges_long_term,balance_sheet_assets_ferc1,yes,,,,Ryan/David spoke - believe derivatives & hedges use customer funds to provide benefit - just like physical assets. +special_funds_(non_major_only),special_funds_(non_major_only),special_funds_all,balance_sheet_assets_ferc1,yes,,,, +special_funds_(non_major_only),special_funds_(non_major_only),other_special_funds,balance_sheet_assets_ferc1,yes,,,, +special_funds_(non_major_only),special_funds_(non_major_only),special_funds,balance_sheet_assets_ferc1,yes,,,, +special_funds_(non_major_only),special_funds_(non_major_only),special_deposits,balance_sheet_assets_ferc1,yes,,,, +(less)_capital_stock_expense,(less)_capital_stock_expense,capital_stock_expense,balance_sheet_liabilities_ferc1,no,,,, +(less)_discount_on_capital_stock,(less)_discount_on_capital_stock,discount_on_capital_stock,balance_sheet_liabilities_ferc1,no,,,, +(less)_reaquired_capital_stock,(less)_reaquired_capital_stock,reacquired_capital_stock,balance_sheet_liabilities_ferc1,no,,,, +_noncorporate_proprietorship_(non_major_only),_noncorporate_proprietorship_(non_major_only),noncorporate_proprietorship,balance_sheet_liabilities_ferc1,no,,,, +appropriated_retained_earnings_amort_reserve_federal,appropriated_retained_earnings_amort_reserve_federal,appropriated_retained_earnings_amortization_reserve_federal,retained_earnings_ferc1,no,,,, +depreciation,depreciation,accumulated_depreciation,electric_plant_depreciation_functional_ferc1,no,,future,, +depreciation,depreciation,accumulated_depreciation,electric_plant_depreciation_functional_ferc1,yes,,in_service,, +depreciation,depreciation,accumulated_depreciation,electric_plant_depreciation_functional_ferc1,yes,,leased,, +depreciation,depreciation,accumulated_depreciation,electric_plant_depreciation_functional_ferc1,partial,,total,, diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index ddaa2246e3..8b29c4bcbe 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -446,8 +446,10 @@ def align_row_numbers_dbf(df: pd.DataFrame, params: AlignRowNumbersDbf) -> pd.Da logger.info( f"Aligning row numbers from DBF row to XBRL map for {params.dbf_table_names}" ) - row_map = read_dbf_to_xbrl_map(dbf_table_names=params.dbf_table_names).pipe( - fill_dbf_to_xbrl_map + row_map = ( + read_dbf_to_xbrl_map(dbf_table_names=params.dbf_table_names) + .pipe(fill_dbf_to_xbrl_map) + .drop(columns=["sched_table_name", "row_literal"]) ) if row_map.isnull().any(axis=None): raise ValueError( @@ -458,7 +460,8 @@ def align_row_numbers_dbf(df: pd.DataFrame, params: AlignRowNumbersDbf) -> pd.Da df = pd.merge(df, row_map, on=["report_year", "row_number"], how="left") if df.xbrl_factoid.isna().any(): raise ValueError( - rf"Found null row labeles after aligning DBF/XBRL rows. n\ {df[df.xbrl_factoid.isna()]}" + "Found null row labels after aligning DBF/XBRL rows.\n" + f"{df[df.xbrl_factoid.isna()]}" ) # eliminate the header rows since they (should!) contain no data in either the # DBF or XBRL records: @@ -1041,6 +1044,11 @@ def wide_to_tidy_value_types(self) -> list[str]: """Compile a list of all of the ``value_types`` from ``wide_to_tidy``.""" return self.wide_to_tidy.value_types + @property + def aligned_dbf_table_names(self) -> list[str]: + """The list of DBF tables aligned by row number in this transform.""" + return self.align_row_numbers_dbf.dbf_table_names + ################################################################################ # FERC 1 transform helper functions. Probably to be integrated into a class @@ -1107,6 +1115,7 @@ def read_dbf_to_xbrl_map(dbf_table_names: list[str]) -> pd.DataFrame: "report_year", "row_number", "row_type", + "row_literal", "xbrl_factoid", ], ) @@ -1197,11 +1206,11 @@ def fill_dbf_to_xbrl_map( # Forward fill missing XBRL column names, until a new definition for the row # number is encountered: - df["xbrl_factoid"] = df.groupby( + df.loc[:, ["xbrl_factoid", "row_literal"]] = df.groupby( ["row_number", "sched_table_name"] - ).xbrl_factoid.transform("ffill") + )[["xbrl_factoid", "row_literal"]].transform("ffill") # Drop NA values produced in the broadcasting merge onto the exhaustive index. - df = df.dropna(subset="xbrl_factoid").drop(columns=["sched_table_name"]) + df = df.dropna(subset="xbrl_factoid") # There should be no NA values left at this point: if df.isnull().any(axis=None): raise ValueError( diff --git a/test/unit/transform/ferc1_test.py b/test/unit/transform/ferc1_test.py index ccdf2d4193..b366a4ceb4 100644 --- a/test/unit/transform/ferc1_test.py +++ b/test/unit/transform/ferc1_test.py @@ -99,30 +99,28 @@ def test_fill_dbf_to_xbrl_map(): expected = pd.read_csv( StringIO( """ -report_year,row_number,xbrl_factoid -2000,2,account_a -2000,3,account_b -2000,5,account_c -2001,2,account_a -2001,3,account_b -2001,5,account_c -2002,2,account_a -2002,3,account_b -2002,4,account_b1 -2002,6,account_c -2003,2,account_a -2003,3,account_b -2003,4,account_b1 -2003,6,account_c -""" - ) - ) - test_map = TEST_DBF_XBRL_MAP.drop(["row_literal"], axis="columns").reset_index( - drop=True - ) +sched_table_name,report_year,row_literal,row_number,xbrl_factoid +test_table1,2000,"Account A",2,account_a +test_table1,2000,"Account B",3,account_b +test_table1,2000,"Account C",5,account_c +test_table1,2001,"Account A",2,account_a +test_table1,2001,"Account B",3,account_b +test_table1,2001,"Account C",5,account_c +test_table1,2002,"Account A",2,account_a +test_table1,2002,"Account B",3,account_b +test_table1,2002,"Account B1",4,account_b1 +test_table1,2002,"Account C",6,account_c +test_table1,2003,"Account A",2,account_a +test_table1,2003,"Account B",3,account_b +test_table1,2003,"Account B1",4,account_b1 +test_table1,2003,"Account C",6,account_c +""" + ) + ) + test_map = TEST_DBF_XBRL_MAP.reset_index(drop=True) actual = fill_dbf_to_xbrl_map(df=test_map, dbf_years=sorted(range(2000, 2004))) actual = actual[actual.xbrl_factoid != "HEADER_ROW"].reset_index(drop=True) - pd.testing.assert_frame_equal(actual, expected) + pd.testing.assert_frame_equal(actual, expected, check_like=True) def test_two_table_fill_dbf_to_xbrl_map(): @@ -130,40 +128,38 @@ def test_two_table_fill_dbf_to_xbrl_map(): expected = pd.read_csv( StringIO( """ -report_year,row_number,xbrl_factoid -2000,2,account_a -2000,3,account_b -2000,5,account_c -2000,7,account_d -2000,8,account_e -2001,2,account_a -2001,3,account_b -2001,5,account_c -2001,7,account_d -2001,8,account_e -2002,2,account_a -2002,3,account_b -2002,4,account_b1 -2002,6,account_c -2002,8,account_d -2002,9,account_e -2003,2,account_a -2003,3,account_b -2003,4,account_b1 -2003,6,account_c -2003,8,account_d -2003,9,account_e -""" - ) - ) - test_map = ( - pd.concat([TEST_DBF_XBRL_MAP, TEST_MUTLI_TABLE_DBF_XBRL_MAP]) - .drop(["row_literal"], axis="columns") - .reset_index(drop=True) - ) +sched_table_name,report_year,row_number,xbrl_factoid,row_literal +test_table1,2000,2,account_a,"Account A" +test_table1,2000,3,account_b,"Account B" +test_table1,2000,5,account_c,"Account C" +test_table2,2000,7,account_d,"Account D" +test_table2,2000,8,account_e,"Account E" +test_table1,2001,2,account_a,"Account A" +test_table1,2001,3,account_b,"Account B" +test_table1,2001,5,account_c,"Account C" +test_table2,2001,7,account_d,"Account D" +test_table2,2001,8,account_e,"Account E" +test_table1,2002,2,account_a,"Account A" +test_table1,2002,3,account_b,"Account B" +test_table1,2002,4,account_b1,"Account B1" +test_table1,2002,6,account_c,"Account C" +test_table2,2002,8,account_d,"Account D" +test_table2,2002,9,account_e,"Account E" +test_table1,2003,2,account_a,"Account A" +test_table1,2003,3,account_b,"Account B" +test_table1,2003,4,account_b1,"Account B1" +test_table1,2003,6,account_c,"Account C" +test_table2,2003,8,account_d,"Account D" +test_table2,2003,9,account_e,"Account E" +""" + ) + ) + test_map = pd.concat( + [TEST_DBF_XBRL_MAP, TEST_MUTLI_TABLE_DBF_XBRL_MAP] + ).reset_index(drop=True) actual = fill_dbf_to_xbrl_map(df=test_map, dbf_years=sorted(range(2000, 2004))) actual = actual[actual.xbrl_factoid != "HEADER_ROW"].reset_index(drop=True) - pd.testing.assert_frame_equal(actual, expected) + pd.testing.assert_frame_equal(actual, expected, check_like=True) WIDE_TO_TIDY_DF = pd.read_csv( diff --git a/tox.ini b/tox.ini index 7e98f2903a..31bbacff41 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ commands = pre-commit run --all-files --show-diff-on-failure name-tests-test [testenv:linters] -description = Run the pre-commit, and ruff linters. +description = Run the pre-commit and ruff linters skip_install = false extras = {[testenv:pre_commit]extras}