Merge pull request #45 from sedos-project/merge-and-write-periods

Merge and write periods

data_adapter_oemof/adapters.py

@@ -12,7 +12,7 @@
 
 class Adapter:
     type: str = "adapter"
-    extra_attributes = ("name", "type")
+    extra_attributes = ("name", "type", "year")
 
     def as_dict(self):
         """
@@ -45,15 +45,25 @@ def get_default_parameters(cls, struct: dict, mapper: Mapper) -> dict:
         mapped_values = mapper.get_default_mappings(struct)
         defaults.update(mapped_values)
         # Add additional attributes
+
         attributes = {
-            "name": calculations.get_name(
-                mapper.get("region"), mapper.get("year"), mapper.get("tech")
-            ),
             "region": mapper.get("region"),
             "year": mapper.get("year"),
         }
         defaults.update(attributes)
 
+        # add name if found in data, else use calculation for name:
+        if (name := mapper.get("name")) is not None:
+            defaults.update({"name": name})
+        else:
+            defaults.update(
+                {
+                    "name": calculations.get_name(
+                        mapper.get("region"), mapper.get("carrier"), mapper.get("tech")
+                    )
+                }
+            )
+
         return defaults
 
 

data_adapter_oemof/build_datapackage.py

@@ -1,6 +1,7 @@
 import dataclasses
 import os
 import warnings
+from typing import Union
 
 import pandas as pd
 from data_adapter import core
@@ -72,6 +73,54 @@ def refactor_timeseries(timeseries: pd.DataFrame):
     return df_timeseries
 
 
+# Define a function to aggregate differing values into a list
+def _listify_to_periodic(group_df) -> pd.Series:
+    """
+    Method to aggregate scalar values to periodical values grouped by "name"
+    For each group, check whether scalar values differ over the years.
+    If yes, write as lists, if not, the original value is written.
+
+    If there is no "year" column, assume the data is already aggregated and
+    pass as given.
+
+    Parameters
+    ----------
+    group_df
+
+    Returns
+    -------
+
+    """
+
+    if "year" not in group_df.columns:
+        return group_df
+
+    unique_values = pd.Series()
+    for col in group_df.columns:  # Exclude 'name' column
+        if isinstance(group_df[col][group_df.index[0]], dict):
+            # Unique input/output parameters are not allowed per period
+            unique_values[col] = group_df[col][group_df.index[0]]
+            continue
+        # Lists and Series can be passed for special Facades only.
+        # Sequences shall be passed as sequences (via links.csv):
+        elif any(
+            [
+                isinstance(col_entry, Union[list, pd.Series])
+                for col_entry in group_df[col]
+            ]
+        ):
+            values = group_df[col].explode().unique()
+        else:
+            values = group_df[col].unique()
+        if len(values) > 1:
+            unique_values[col] = list(group_df[col])
+        else:
+            unique_values[col] = group_df[col].iloc[0]
+    unique_values["name"] = "_".join(group_df.name)
+    unique_values.drop("year")
+    return unique_values
+
+
 @dataclasses.dataclass
 class DataPackage:
     parametrized_elements: dict[
@@ -229,6 +278,37 @@ def save_datapackage_to_csv(self, destination: str) -> None:
 
         return None
 
+    @staticmethod
+    def yearly_scalars_to_periodic_values(scalar_dataframe) -> None:
+        """
+        Turns yearly scalar values to periodic values
+
+        First searches for the sequence length which is the length of the complete sequence.
+
+        Then iterates for every element in parametrized elements, groups them for name
+        then applies aggregation method
+
+        Returns None
+        -------
+
+        """
+        identifiers = ["region", "carrier", "tech"]
+        # Check if the identifiers exist if not they will be omitted
+        for poss, existing in enumerate(
+            [id in scalar_dataframe.columns for id in identifiers]
+        ):
+            if existing:
+                continue
+            else:
+                scalar_dataframe[identifiers[poss]] = identifiers[poss]
+
+        scalar_dataframe = (
+            scalar_dataframe.groupby(["region", "carrier", "tech"])
+            .apply(lambda x: _listify_to_periodic(x))
+            .reset_index(drop=True)
+        )
+        return scalar_dataframe
+
     @classmethod
     def build_datapackage(cls, adapter: Adapter):
         """
@@ -253,12 +333,22 @@ def build_datapackage(cls, adapter: Adapter):
         for process_name, struct in es_structure.items():
             process_data = adapter.get_process(process_name)
             timeseries = process_data.timeseries
+            if isinstance(timeseries.columns, pd.MultiIndex):
+                # FIXME: Will Regions be lists of strings or strings?
+                timeseries.columns = (
+                    timeseries.columns.get_level_values(0)
+                    + "_"
+                    + [x[0] for x in timeseries.columns.get_level_values(1).values]
+                )
             facade_adapter_name: str = PROCESS_TYPE_MAP[process_name]
             facade_adapter = FACADE_ADAPTERS[facade_adapter_name]
             components = []
             process_busses = []
+            process_scalars = cls.yearly_scalars_to_periodic_values(
+                process_data.scalars
+            )
             # Build class from adapter with Mapper and add up for each component within the Element
-            for component_data in process_data.scalars.to_dict(orient="records"):
+            for component_data in process_scalars.to_dict(orient="records"):
                 component_mapper = Mapper(
                     adapter=facade_adapter,
                     process_name=process_name,
@@ -285,6 +375,7 @@ def build_datapackage(cls, adapter: Adapter):
             parametrized_elements[process_name] = pd.DataFrame(components)
 
             parametrized_sequences = {process_name: timeseries}
+        # Create Bus Element from all unique `busses` found in elements
         parametrized_elements["bus"] = pd.DataFrame(
             {
                 "name": (names := pd.unique(parametrized_elements["bus"])),

tests/_files/build_datapackage_goal/data/elements/modex_tech_generator_gas.csv

@@ -1,4 +1,2 @@
-from_bus;to_bus;carrier;tech;capacity;efficiency;marginal_cost;carrier_cost;capacity_cost;expandable;capacity_potential;capacity_minimum;input_parameters;output_parameters;name;type
-ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB-2016-generator_gas;conversion
-ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB-2030-generator_gas;conversion
-ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB-2050-generator_gas;conversion
+from_bus;to_bus;carrier;tech;capacity;efficiency;marginal_cost;carrier_cost;capacity_cost;expandable;capacity_potential;capacity_minimum;input_parameters;output_parameters;name;type;year
+ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB_gas_generator_gas;conversion;[2016, 2030, 2050]

tests/_files/build_datapackage_goal/data/elements/modex_tech_storage_battery.csv

@@ -1,4 +1,2 @@
-bus;carrier;tech;storage_capacity;capacity;capacity_cost;storage_capacity_cost;storage_capacity_potential;capacity_potential;expandable;lifetime;age;fixed_costs;marginal_cost;efficiency;input_parameters;output_parameters;name;type
-electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[];0;1;{};{};BB-2016-storage_battery;storage
-electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[];0;1;{};{};BB-2030-storage_battery;storage
-electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[];0;1;{};{};BB-2050-storage_battery;storage
+bus;carrier;tech;storage_capacity;capacity;capacity_cost;storage_capacity_cost;storage_capacity_potential;capacity_potential;expandable;lifetime;age;fixed_costs;marginal_cost;efficiency;input_parameters;output_parameters;name;type;year
+electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[1, 2, 3];0;1;{};{};BB_Lithium_storage_battery;storage;[2016, 2030, 2050]

tests/_files/build_datapackage_goal/data/elements/modex_tech_wind_turbine_onshore.csv

@@ -1,4 +1,2 @@
-bus;carrier;tech;profile;capacity;capacity_potential;capacity_minimum;expandable;marginal_cost;capacity_cost;lifetime;age;fixed_costs;output_parameters;name;type
-electricity;wind;wind_turbine_onshore;onshore_BB;5700.03;inf;;False;0;;25.4;0;23280.0;{};BB-2016-wind_turbine_onshore;volatile
-electricity;wind;wind_turbine_onshore;onshore_BB;5700.03375;inf;;False;0;;30.0;0;12600.0;{};BB-2030-wind_turbine_onshore;volatile
-electricity;wind;wind_turbine_onshore;onshore_BB;5700.03375;inf;;False;0;;30.0;0;11340.0;{};BB-2050-wind_turbine_onshore;volatile
+bus;carrier;tech;profile;capacity;capacity_potential;capacity_minimum;expandable;marginal_cost;capacity_cost;lifetime;age;fixed_costs;output_parameters;name;type;year
+electricity;wind;wind_turbine_onshore;onshore_BB;[5700.03, 5700.03375, 5700.03375];inf;;False;0;;[25.4, 30.0, 30.0];0;[23280.0, 12600.0, 11340.0];{};BB_wind_wind_turbine_onshore;volatile;[2016, 2030, 2050]

tests/_files/build_datapackage_goal/datapackage.json

@@ -162,6 +162,11 @@
             "format": "default",
             "name": "type",
             "type": "string"
+          },
+          {
+            "format": "default",
+            "name": "year",
+            "type": "array"
           }
         ],
         "foreignKeys": [
@@ -267,6 +272,11 @@
             "format": "default",
             "name": "type",
             "type": "string"
+          },
+          {
+            "format": "default",
+            "name": "year",
+            "type": "array"
           }
         ],
         "foreignKeys": [
@@ -323,7 +333,7 @@
           {
             "format": "default",
             "name": "capacity",
-            "type": "number"
+            "type": "array"
           },
           {
             "format": "default",
@@ -353,7 +363,7 @@
           {
             "format": "default",
             "name": "lifetime",
-            "type": "number"
+            "type": "array"
           },
           {
             "format": "default",
@@ -363,7 +373,7 @@
           {
             "format": "default",
             "name": "fixed_costs",
-            "type": "number"
+            "type": "array"
           },
           {
             "format": "default",
@@ -379,6 +389,11 @@
             "format": "default",
             "name": "type",
             "type": "string"
+          },
+          {
+            "format": "default",
+            "name": "year",
+            "type": "array"
           }
         ],
         "foreignKeys": [

tests/test_build_datapackage.py

@@ -122,7 +122,11 @@ def mock_get_process(process_name):
                         1: "gas",
                         2: "gas",
                     },
-                    "condensing_efficiency": {0: 0.85, 1: 0.85, 2: 0.9},
+                    "condensing_efficiency": {
+                        0: 0.16,
+                        1: 0.3,
+                        2: 0.5,
+                    },
                     "electric_efficiency": {0: 0.35, 1: 0.35, 2: 0.4},
                     "thermal_efficiency": {0: 0.5, 1: 0.5, 2: 0.45},
                 }