Bug 509

src/pudl/metadata/classes.py

@@ -1676,6 +1676,23 @@ def encode(self, df: pd.DataFrame) -> pd.DataFrame:
                 )
         return df
 
+    def enforce_schema(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Drop columns not in the DB schema and enforce specified types."""
+        expected_cols = pd.Index(self.get_field_names())
+        missing_cols = list(expected_cols.difference(df.columns))
+        if missing_cols:
+            raise ValueError(
+                f"{self.name}: Missing columns found when enforcing table "
+                f"schema: {missing_cols}"
+            )
+        df = self.format_df(df)
+        pk = self.schema.primary_key
+        if pk and not df[df.duplicated(subset=pk)].empty:
+            raise ValueError(
+                f"{self.name} Duplicate primary keys when enforcing schema."
+            )
+        return df
+
 
 # ---- Package ---- #
 

src/pudl/package_data/eia923/column_maps/generation_fuel.csv

@@ -3,7 +3,7 @@ plant_id_eia,plant_id,plant_id,plant_id,plant_id,plant_id,plant_id,plant_id,plan
 combined_heat_power,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_power_plant,combined_heat_and_power_plant,combined_heat_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant,combined_heat_and_power_plant
 nuclear_unit_id,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_i_d,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id,nuclear_unit_id
 plant_name_eia,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name
-operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name
+utility_name_eia,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name,operator_name
 operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id,operator_id
 plant_state,state,state,state,state,state,state,state,state,state,state,state,plant_state,state,plant_state,plant_state,plant_state,plant_state,plant_state,plant_state,plant_state,plant_state
 census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region,census_region

src/pudl/transform/classes.py

@@ -1263,17 +1263,5 @@ def enforce_schema(self, df: pd.DataFrame) -> pd.DataFrame:
         """Drop columns not in the DB schema and enforce specified types."""
         logger.info(f"{self.table_id.value}: Enforcing database schema on dataframe.")
         resource = Package.from_resource_ids().get_resource(self.table_id.value)
-        expected_cols = pd.Index(resource.get_field_names())
-        missing_cols = list(expected_cols.difference(df.columns))
-        if missing_cols:
-            raise ValueError(
-                f"{self.table_id.value}: Missing columns found when enforcing table "
-                f"schema: {missing_cols}"
-            )
-        df = resource.format_df(df)
-        pk = resource.schema.primary_key
-        if pk and not df[df.duplicated(subset=pk)].empty:
-            raise ValueError(
-                f"{self.table_id.value} Duplicate primary keys when enforcing schema."
-            )
+        df = resource.enforce_schema(df)
         return df

src/pudl/transform/eia.py

@@ -1170,6 +1170,17 @@ def transform(
         "boilers_annual_eia",
     )
 
+    # Remove fields that came from input data but aren't in the
+    # corresponding SQLite tables. The data may still exist but has been
+    # moved elsewhere.
+    for cat in eia_transformed_dfs:
+        resource = (
+            pudl.metadata.classes.Package.from_resource_ids().
+            get_resource(cat)
+        )
+        eia_transformed_dfs[cat] = resource.enforce_schema(
+            eia_transformed_dfs[cat])
+
     eia_transformed_dfs["plants_eia860"] = fillna_balancing_authority_codes_via_names(
         df=eia_transformed_dfs["plants_eia860"]
     ).pipe(

src/pudl/transform/eia923.py

@@ -617,8 +617,6 @@ def generation_fuel(eia923_dfs, eia923_transformed_dfs):
     # Drop fields we're not inserting into the generation_fuel_eia923 table.
     cols_to_drop = [
         "combined_heat_power",
-        "plant_name_eia",
-        "operator_name",
         "operator_id",
         "plant_state",
         "census_region",
@@ -1072,7 +1070,6 @@ def fuel_receipts_costs(eia923_dfs, eia923_transformed_dfs):
     # Drop fields we're not inserting into the fuel_receipts_costs_eia923
     # table.
     cols_to_drop = [
-        "plant_name_eia",
         "plant_state",
         "operator_name",
         "operator_id",