From cae03c7aaeea69b66cc6cf1f591f84b8a15cef08 Mon Sep 17 00:00:00 2001 From: FelixMau Date: Wed, 28 Jun 2023 12:38:48 +0200 Subject: [PATCH 1/5] Adding bus to parametrized elements iterative Adding busses by calling get_bus from Mapper. Also removed `cls` parameter from get_default and get bus since the class is stored in Mapper as `adapter` anyway. --- data_adapter_oemof/adapters.py | 2 +- data_adapter_oemof/build_datapackage.py | 19 ++++++++++++++++--- data_adapter_oemof/mappings.py | 16 +++++++--------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/data_adapter_oemof/adapters.py b/data_adapter_oemof/adapters.py index b2b3df5..84d7f9d 100644 --- a/data_adapter_oemof/adapters.py +++ b/data_adapter_oemof/adapters.py @@ -46,7 +46,7 @@ def get_default_parameters( "type": cls.type, } # Add mapped attributes - mapped_values = mapper.get_default_mappings(cls, struct) + mapped_values = mapper.get_default_mappings(struct) defaults.update(mapped_values) # Add additional attributes attributes = { diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index 3a90730..26e5549 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -6,7 +6,7 @@ from data_adapter import core from data_adapter.preprocessing import Adapter from data_adapter_oemof.adapters import FACADE_ADAPTERS -from data_adapter_oemof.mappings import PROCESS_TYPE_MAP +from data_adapter_oemof.mappings import PROCESS_TYPE_MAP, Mapper def refactor_timeseries(timeseries: pd.DataFrame): @@ -133,21 +133,30 @@ def build_datapackage(cls, adapter: Adapter): """ es_structure = adapter.get_structure() - parametrized_elements = {} + parametrized_elements = {"bus":[]} parametrized_sequences = {} foreign_keys = {} + # Iterate Elements for process_name, struct in es_structure.items(): process_data = adapter.get_process(process_name) timeseries = refactor_timeseries(process_data.timeseries) facade_adapter_name: str = PROCESS_TYPE_MAP[process_name] facade_adapter = FACADE_ADAPTERS[facade_adapter_name] components = [] + + # Build class from adapter with Mapper and add up for each component within the Element for component_data in process_data.scalars.to_dict(orient="records"): component = facade_adapter.parametrize_dataclass( process_name, component_data, timeseries, struct ) components.append(component.as_dict()) + # Fill bus.csv with all busses occurring + parametrized_elements["bus"] += list(Mapper(adapter=facade_adapter, + process_name=process_name, + data=component_data, + timeseries=timeseries).get_busses(struct).values()) + scalars = pd.DataFrame(components) # check if facade_adapter already exists @@ -163,7 +172,11 @@ def build_datapackage(cls, adapter: Adapter): parametrized_elements[facade_adapter_name] = scalars parametrized_sequences = {process_name: timeseries} - parametrized_elements = add_bus_to_element_dict(parametrized_elements) + parametrized_elements["bus"] = pd.DataFrame( + {"name": pd.unique(parametrized_elements["bus"]), + "type": ["bus" for i in pd.unique(parametrized_elements["bus"])], + "blanced": [True for i in pd.unique(parametrized_elements["bus"])]} + ) return cls( parametrized_elements=parametrized_elements, parametrized_sequences=parametrized_sequences, diff --git a/data_adapter_oemof/mappings.py b/data_adapter_oemof/mappings.py index 7d7fb02..3a232f8 100644 --- a/data_adapter_oemof/mappings.py +++ b/data_adapter_oemof/mappings.py @@ -124,7 +124,7 @@ def get(self, key, field_type: Optional[Type] = None): mapped_key = self.map_key(key) return self.get_data(mapped_key, field_type) - def get_busses(self, cls, struct): + def get_busses(self, struct): """ Identify mentioned buses in the facade. Determine if each bus in the facade is classified as an "input"/"output". @@ -137,16 +137,15 @@ def get_busses(self, cls, struct): Between the structure CSV and the adapter's buses take name from the structure. :param parameter: paramter for mapping different parameters within a process - :param cls: Child from Adapter class :param struct: dict :return: dictionary with tabular like Busses """ bus_occurrences_in_fields = [ - field.name for field in dataclasses.fields(cls) if "bus" in field.name + field.name for field in dataclasses.fields(self.adapter) if "bus" in field.name ] if len(bus_occurrences_in_fields) == 0: logger.warning( - f"No busses found in facades fields for Dataadapter {cls.__name__}" + f"No busses found in facades fields for Dataadapter {self.adapter.__name__}" ) # { input: [], output :[]} # {default: {}, @@ -155,7 +154,7 @@ def get_busses(self, cls, struct): for bus in bus_occurrences_in_fields: # emission_bus # 1. Check for existing mappings try: - bus_dict[bus] = self.bus_map[cls.__name__][bus] + bus_dict[bus] = self.bus_map[self.adapter.__name__][bus] continue except KeyError: pass @@ -201,20 +200,19 @@ def get_busses(self, cls, struct): return bus_dict - def get_default_mappings(self, cls, struct): + def get_default_mappings(self, struct): """ :param struct: dict - :param cls: Data-adapter which is inheriting from oemof.tabular facade :param mapper: Mapper to map oemof.tabular data names to Project naming :return: Dictionary for all fields that the facade can take and matching data """ mapped_all_class_fields = { field.name: value - for field in dataclasses.fields(cls) + for field in dataclasses.fields(self.adapter) if (value := self.get(field.name, field.type)) is not None } - mapped_all_class_fields.update(self.get_busses(cls, struct)) + mapped_all_class_fields.update(self.get_busses(struct)) return mapped_all_class_fields @staticmethod From c1571a1884f9e609416e1adb258eb63bf35882de Mon Sep 17 00:00:00 2001 From: FelixMau Date: Wed, 28 Jun 2023 12:43:13 +0200 Subject: [PATCH 2/5] Cleaning --- data_adapter_oemof/build_datapackage.py | 26 +++++++++++++++++-------- data_adapter_oemof/mappings.py | 14 ++++++++----- tests/test_mapping.py | 10 ++++++---- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index 26e5549..d7e0548 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -63,6 +63,7 @@ def refactor_timeseries(timeseries: pd.DataFrame): return df_timeseries + def add_bus_to_element_dict(elements_dict): """ Takes a sequence dict of processes with sequence data and searches for busses in columns that contain `bus` @@ -86,6 +87,7 @@ def add_bus_to_element_dict(elements_dict): return elements_dict + @dataclasses.dataclass class DataPackage: parametrized_elements: dict # datadict with scalar data in form of {type:pd.DataFrame(type)} @@ -133,7 +135,7 @@ def build_datapackage(cls, adapter: Adapter): """ es_structure = adapter.get_structure() - parametrized_elements = {"bus":[]} + parametrized_elements = {"bus": []} parametrized_sequences = {} foreign_keys = {} # Iterate Elements @@ -152,10 +154,16 @@ def build_datapackage(cls, adapter: Adapter): components.append(component.as_dict()) # Fill bus.csv with all busses occurring - parametrized_elements["bus"] += list(Mapper(adapter=facade_adapter, - process_name=process_name, - data=component_data, - timeseries=timeseries).get_busses(struct).values()) + parametrized_elements["bus"] += list( + Mapper( + adapter=facade_adapter, + process_name=process_name, + data=component_data, + timeseries=timeseries, + ) + .get_busses(struct) + .values() + ) scalars = pd.DataFrame(components) @@ -173,9 +181,11 @@ def build_datapackage(cls, adapter: Adapter): parametrized_sequences = {process_name: timeseries} parametrized_elements["bus"] = pd.DataFrame( - {"name": pd.unique(parametrized_elements["bus"]), - "type": ["bus" for i in pd.unique(parametrized_elements["bus"])], - "blanced": [True for i in pd.unique(parametrized_elements["bus"])]} + { + "name": pd.unique(parametrized_elements["bus"]), + "type": ["bus" for i in pd.unique(parametrized_elements["bus"])], + "blanced": [True for i in pd.unique(parametrized_elements["bus"])], + } ) return cls( parametrized_elements=parametrized_elements, diff --git a/data_adapter_oemof/mappings.py b/data_adapter_oemof/mappings.py index 3a232f8..fa07bfa 100644 --- a/data_adapter_oemof/mappings.py +++ b/data_adapter_oemof/mappings.py @@ -141,7 +141,9 @@ def get_busses(self, struct): :return: dictionary with tabular like Busses """ bus_occurrences_in_fields = [ - field.name for field in dataclasses.fields(self.adapter) if "bus" in field.name + field.name + for field in dataclasses.fields(self.adapter) + if "bus" in field.name ] if len(bus_occurrences_in_fields) == 0: logger.warning( @@ -151,7 +153,7 @@ def get_busses(self, struct): # {default: {}, # co2:{}}} bus_dict = {} - for bus in bus_occurrences_in_fields: # emission_bus + for bus in bus_occurrences_in_fields: # emission_bus # 1. Check for existing mappings try: bus_dict[bus] = self.bus_map[self.adapter.__name__][bus] @@ -159,15 +161,17 @@ def get_busses(self, struct): except KeyError: pass - #TODO: Make use of Parameter [stuct.csv]? + # TODO: Make use of Parameter [stuct.csv]? # Do we need parameter specific Bus structure? Maybe for multiple in/output? if len(struct.keys()) == 1: struct = list(struct.values())[0] elif "default" in struct.keys(): struct = struct["default"] else: - warnings.warn("Please check structure and provide either one set of inputs/outputs or specify as default" - "Parameter specific busses not implemented yet") + warnings.warn( + "Please check structure and provide either one set of inputs/outputs or specify as default" + "Parameter specific busses not implemented yet" + ) # 2. Check for default busses if bus in ("bus", "from_bus", "to_bus"): diff --git a/tests/test_mapping.py b/tests/test_mapping.py index 4d44611..fc27fd0 100644 --- a/tests/test_mapping.py +++ b/tests/test_mapping.py @@ -107,7 +107,7 @@ def test_get_busses(): } } - struct = {"default":{"inputs": ["ch4"], "outputs": ["electricity", "heat"]}} + struct = {"default": {"inputs": ["ch4"], "outputs": ["electricity", "heat"]}} expected = {"electricity_bus": "electricity", "heat_bus": "heat", "fuel_bus": "ch4"} @@ -135,7 +135,9 @@ def test_default_bus_mapping(): "custom_capacity": 100.0, } - struct = {"default":{"inputs": ["electricity_bus_1"], "outputs": ["electricity_bus_2"]}} + struct = { + "default": {"inputs": ["electricity_bus_1"], "outputs": ["electricity_bus_2"]} + } expected = { "from_bus": "electricity_bus_1", @@ -158,7 +160,7 @@ def test_default_bus_mapping(): "custom_capacity": 100.0, } - struct = {"default":{"inputs": [], "outputs": ["electricity"]}} + struct = {"default": {"inputs": [], "outputs": ["electricity"]}} expected = {"bus": "electricity"} @@ -187,7 +189,7 @@ def test_get_matched_busses(): bus_map = {} - struct = {"default":{"inputs": ["ch4fuel"], "outputs": ["elec", "heating"]}} + struct = {"default": {"inputs": ["ch4fuel"], "outputs": ["elec", "heating"]}} expected = {"electricity_bus": "elec", "heat_bus": "heating", "fuel_bus": "ch4fuel"} From a73393c3cbe1f309ec6686035d3f9f4b4a459b4b Mon Sep 17 00:00:00 2001 From: FelixMau Date: Wed, 28 Jun 2023 12:48:50 +0200 Subject: [PATCH 3/5] Remove old function --- data_adapter_oemof/build_datapackage.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index d7e0548..fc6e973 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -64,30 +64,6 @@ def refactor_timeseries(timeseries: pd.DataFrame): return df_timeseries -def add_bus_to_element_dict(elements_dict): - """ - Takes a sequence dict of processes with sequence data and searches for busses in columns that contain `bus` - :param elements_dict: - :return: Updated Sequence with additional entry "bus" that contains all occurring bus entries as balanced busses. - """ - unique_entries = set() - for element in elements_dict.values(): - bus_occourences = [col for col in element.columns if "bus" in col] - for bus_category_name in bus_occourences: - unique_entries.update(element[bus_category_name].unique()) - - unique_entries = pd.unique(list(unique_entries)) - elements_dict["bus"] = pd.DataFrame( - { - "name": unique_entries, - "type": ["bus" for i in unique_entries], - "balanced": [True for i in unique_entries], - } - ) - - return elements_dict - - @dataclasses.dataclass class DataPackage: parametrized_elements: dict # datadict with scalar data in form of {type:pd.DataFrame(type)} From 6b0f23df5c56ac66ddb55313ef3566328cba23da Mon Sep 17 00:00:00 2001 From: FelixMau Date: Wed, 28 Jun 2023 13:25:05 +0200 Subject: [PATCH 4/5] Learned about walrus (thx) --- data_adapter_oemof/build_datapackage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index fc6e973..85ea5cc 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -158,9 +158,9 @@ def build_datapackage(cls, adapter: Adapter): parametrized_sequences = {process_name: timeseries} parametrized_elements["bus"] = pd.DataFrame( { - "name": pd.unique(parametrized_elements["bus"]), - "type": ["bus" for i in pd.unique(parametrized_elements["bus"])], - "blanced": [True for i in pd.unique(parametrized_elements["bus"])], + "name": (names := pd.unique(parametrized_elements["bus"])), + "type": ["bus" for i in names], + "blanced": [True for i in names], } ) return cls( From 4337b4480e5010e46f9e38d18eab7d44bcff3a1d Mon Sep 17 00:00:00 2001 From: FelixMau Date: Wed, 28 Jun 2023 14:28:14 +0200 Subject: [PATCH 5/5] Bus for each row. --- data_adapter_oemof/build_datapackage.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index 85ea5cc..00e4eb0 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -121,25 +121,24 @@ def build_datapackage(cls, adapter: Adapter): facade_adapter_name: str = PROCESS_TYPE_MAP[process_name] facade_adapter = FACADE_ADAPTERS[facade_adapter_name] components = [] - # Build class from adapter with Mapper and add up for each component within the Element for component_data in process_data.scalars.to_dict(orient="records"): component = facade_adapter.parametrize_dataclass( process_name, component_data, timeseries, struct ) components.append(component.as_dict()) - - # Fill bus.csv with all busses occurring - parametrized_elements["bus"] += list( - Mapper( - adapter=facade_adapter, - process_name=process_name, - data=component_data, - timeseries=timeseries, + component_mapper = Mapper( + adapter=facade_adapter, + process_name=process_name, + data=component_data, + timeseries=timeseries, + ) + # Fill bus.csv with all busses occurring + parametrized_elements["bus"] += list( + component_mapper + .get_busses(struct) + .values() ) - .get_busses(struct) - .values() - ) scalars = pd.DataFrame(components)