From 378fe5cdfa955d754f2417b8f9d356324d53fff7 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 10 Oct 2023 16:08:45 +0200 Subject: [PATCH 01/43] Update pypsa version and remove limits for pyomo and pandas --- setup.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 5491991a..9303239d 100755 --- a/setup.py +++ b/setup.py @@ -54,11 +54,7 @@ def read(*names, **kwargs): "loguru", "matplotlib >= 3.0.3", "oedialect", - # Fix upper version limits for pyomo and pandas - # Related to problems with old pypsa version - "pandas < 2", - "pyomo>6.4, <6.6, !=6.4.3", - "pypsa == 0.20.1", + "pypsa == 0.25.1", "rtree", "saio", "scikit-learn", From 254c6d6d38d073ac546ed726d8d4934eef845b02 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 10 Oct 2023 16:10:03 +0200 Subject: [PATCH 02/43] Chnage names of pypsa modules --- etrago/cluster/electrical.py | 2 +- etrago/cluster/gas.py | 2 +- etrago/cluster/spatial.py | 8 ++++---- etrago/tools/constraints.py | 2 +- etrago/tools/execute.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 44fc5526..831352da 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -27,7 +27,7 @@ import logging from pypsa import Network - from pypsa.networkclustering import ( + from pypsa.clustering.spatial import ( aggregatebuses, aggregategenerators, aggregateoneport, diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index b25f64e6..8d9bf523 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -27,7 +27,7 @@ import logging from pypsa import Network - from pypsa.networkclustering import ( + from pypsa.clustering.spatial import ( aggregatebuses, aggregateoneport, busmap_by_kmeans, diff --git a/etrago/cluster/spatial.py b/etrago/cluster/spatial.py index be78296a..9d122978 100755 --- a/etrago/cluster/spatial.py +++ b/etrago/cluster/spatial.py @@ -30,8 +30,8 @@ import multiprocessing as mp from networkx import NetworkXNoPath - from pypsa.networkclustering import ( - _flatten_multiindex, + from pypsa.clustering.spatial import ( + flatten_multiindex, busmap_by_kmeans, busmap_by_stubs, get_clustering_from_busmap, @@ -237,7 +237,7 @@ def arrange_dc_bus0_bus1(network): strategies = strategies_links() strategies.update(cus_strateg) new_df = links.groupby(grouper, axis=0).agg(strategies) - new_df.index = _flatten_multiindex(new_df.index).rename("name") + new_df.index = flatten_multiindex(new_df.index).rename("name") new_df = pd.concat( [new_df, network.links.loc[~links_agg_b]], axis=0, sort=False ) @@ -257,7 +257,7 @@ def arrange_dc_bus0_bus1(network): weighting.loc[df_agg.columns], axis=1 ) pnl_df = df_agg.groupby(grouper, axis=1).sum() - pnl_df.columns = _flatten_multiindex(pnl_df.columns).rename( + pnl_df.columns = flatten_multiindex(pnl_df.columns).rename( "name" ) new_pnl[attr] = pd.concat( diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index 9785e580..ef532e25 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -27,7 +27,7 @@ from pyomo.environ import Constraint from pypsa.descriptors import expand_series from pypsa.linopt import define_constraints, define_variables, get_var, linexpr -from pypsa.pf import get_switchable_as_dense as get_as_dense +from pypsa.descriptors import get_switchable_as_dense as get_as_dense import numpy as np import pandas as pd import pyomo.environ as po diff --git a/etrago/tools/execute.py b/etrago/tools/execute.py index ea643bf2..b6497a03 100755 --- a/etrago/tools/execute.py +++ b/etrago/tools/execute.py @@ -28,7 +28,7 @@ import time from pypsa.linopf import network_lopf - from pypsa.networkclustering import aggregategenerators + from pypsa.clustering.spatial import aggregategenerators from pypsa.pf import sub_network_pf import numpy as np import pandas as pd From f1383be3d99f1b5458887838d3d09353522c5f19 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 14:52:46 +0200 Subject: [PATCH 03/43] Extend strategies for clustering In pypsa==0.25.2, the clustering strategy is 'consense' per default. That created problems for some columns, which are solved by adding specific strategies for these columns. --- etrago/cluster/spatial.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/etrago/cluster/spatial.py b/etrago/cluster/spatial.py index 9d122978..eac1c257 100755 --- a/etrago/cluster/spatial.py +++ b/etrago/cluster/spatial.py @@ -109,6 +109,18 @@ def sum_with_inf(x): return x.sum() +def strategies_buses(): + return { + "geom": nan_links, + } + + +def strategies_lines(): + return { + "geom": nan_links, + } + + def strategies_one_ports(): return { "StorageUnit": { @@ -129,6 +141,8 @@ def strategies_one_ports(): "e_nom_min": np.sum, "e_nom_max": sum_with_inf, "e_initial": np.sum, + "e_min_pu": np.mean, + "e_max_pu": np.mean, }, } @@ -171,6 +185,11 @@ def strategies_links(): "country": nan_links, "build_year": np.mean, "lifetime": np.mean, + "min_up_time": np.mean, + "min_down_time": np.mean, + "up_time_before": np.mean, + "down_time_before": np.mean, + "committable": np.all, } From 0cf8339669da99098283e014ae1727253d5762df Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 14:54:40 +0200 Subject: [PATCH 04/43] Remove columns from strategies that are not available in the dataframe --- etrago/cluster/spatial.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etrago/cluster/spatial.py b/etrago/cluster/spatial.py index eac1c257..211fc2f8 100755 --- a/etrago/cluster/spatial.py +++ b/etrago/cluster/spatial.py @@ -255,6 +255,9 @@ def arrange_dc_bus0_bus1(network): ) strategies = strategies_links() strategies.update(cus_strateg) + strategies.pop("topo") + strategies.pop("geom") + new_df = links.groupby(grouper, axis=0).agg(strategies) new_df.index = flatten_multiindex(new_df.index).rename("name") new_df = pd.concat( From 9dee433d7f7598cf304336de2df7b142f91910b1 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:18:31 +0200 Subject: [PATCH 05/43] Do not export data as geodataframe --- etrago/tools/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/tools/io.py b/etrago/tools/io.py index c87bda98..b5311f9a 100644 --- a/etrago/tools/io.py +++ b/etrago/tools/io.py @@ -223,7 +223,7 @@ def fetch_by_relname(self, name): vars()[f"egon_etrago_{name.lower()}"].version == self.version ) - df = saio.as_pandas(query, crs=4326).set_index(index) + df = saio.as_pandas(query, crs=4326, geometry=None).set_index(index) if name == "Transformer": df.tap_side = 0 From 91c39e666f182fdf99cf845b940c1f20ea08b1cf Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:20:01 +0200 Subject: [PATCH 06/43] Replace pypsa.aggregate_generators with own aggregation From pypsa>0.25.1 on, the aggregation of already solved networks creates problems in pypsa. It looks for all time series variables in the static dataframe, which leads to key errors, eg. for 'p' of generators. This commit aggeragted the new generators using pandas functions. --- etrago/tools/execute.py | 90 ++++++++++++----------------------------- 1 file changed, 26 insertions(+), 64 deletions(-) diff --git a/etrago/tools/execute.py b/etrago/tools/execute.py index b6497a03..f4ee32fe 100755 --- a/etrago/tools/execute.py +++ b/etrago/tools/execute.py @@ -527,7 +527,7 @@ def import_gen_from_links(network): network.links_t[df].drop( columns=discard_gen.values, inplace=True, errors="ignore" ) - + # Select links that should be represented as generators gas_to_add = network.links[ network.links.carrier.isin( [ @@ -539,74 +539,36 @@ def import_gen_from_links(network): ) ].copy() - # Drop generators from the links table - network.links.drop(gas_to_add.index, inplace=True) - + # Rename bus1 column to bus gas_to_add.rename(columns={"bus1": "bus"}, inplace=True) - # Create generators' names like in network.generators - gas_to_add["Generator"] = ( - gas_to_add["bus"] + " " + gas_to_add.index + gas_to_add["carrier"] + # Aggregate new generators per bus and carrier + df = pd.DataFrame() + df["p_nom"] = gas_to_add.groupby(["bus", "carrier"]).p_nom.sum() + df["p_nom_opt"] = gas_to_add.groupby(["bus", "carrier"]).p_nom_opt.sum() + df["marginal_cost"] = gas_to_add.groupby( + ["bus", "carrier"] + ).marginal_cost.mean() + df["efficiency"] = gas_to_add.groupby(["bus", "carrier"]).efficiency.mean() + df["control"] = "PV" + df.reset_index(inplace=True) + df.index = df.bus + " " + df.carrier + + # Aggregate disptach time series for new generators + gas_to_add["bus1_carrier"] = gas_to_add.bus + " " + gas_to_add.carrier + df_t = ( + network.links_t.p1[gas_to_add.index] + .groupby(gas_to_add.bus1_carrier, axis=1) + .sum() + * -1 ) - gas_to_add_orig = gas_to_add.copy() - gas_to_add.set_index("Generator", drop=True, inplace=True) - gas_to_add = gas_to_add[ - gas_to_add.columns[gas_to_add.columns.isin(network.generators.columns)] - ] - - network.import_components_from_dataframe(gas_to_add, "Generator") - - # Dealing with generators_t - columns_new = network.links_t.p1.columns[ - network.links_t.p1.columns.isin(gas_to_add_orig.index) - ] - - new_gen_t = network.links_t.p1[columns_new] * -1 - new_gen_t.rename(columns=gas_to_add_orig["Generator"], inplace=True) - network.generators_t.p = network.generators_t.p.join(new_gen_t) - - # Drop generators from the links_t table - for df in network.links_t: - if not network.links_t[df].empty: - network.links_t[df].drop( - columns=gas_to_add_orig.index, - inplace=True, - errors="ignore", - ) - - # Group generators per bus if needed - if not ( - network.generators.groupby(["bus", "carrier"]).p_nom.count() == 1 - ).all(): - network.generators["weight"] = network.generators.p_nom - df, df_t = aggregategenerators( - network, - busmap=pd.Series( - index=network.buses.index, data=network.buses.index - ), - custom_strategies=strategies_generators(), - ) - - # Keep control arguments from generators - control = network.generators.groupby( - ["bus", "carrier"] - ).control.first() - control.index = ( - control.index.get_level_values(0) - + " " - + control.index.get_level_values(1) - ) - df.control = control - - # Drop non-aggregated generators - network.mremove("Generator", network.generators.index) - # Insert aggregated generators and time series - network.import_components_from_dataframe(df, "Generator") + # Insert aggregated generators their dispatch time series + network.madd("Generator", df.index, **df) + network.import_series_from_dataframe(df_t, "Generator", "p") - for attr, data in df_t.items(): - if not data.empty: - network.import_series_from_dataframe(data, "Generator", attr) + # Drop links now modelled as generator + network.mremove("Link", gas_to_add.index) return From 9ef6162d3b967909f8bef3cd68aa9f59674098e4 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:23:10 +0200 Subject: [PATCH 07/43] Import network from pypsa.components --- etrago/cluster/gas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 8d9bf523..5b8a07e3 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -26,7 +26,7 @@ if "READTHEDOCS" not in os.environ: import logging - from pypsa import Network + from pypsa.components import Network from pypsa.clustering.spatial import ( aggregatebuses, aggregateoneport, From fb19bb0586132d692ff9a58ce0322d16690aa941 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:24:05 +0200 Subject: [PATCH 08/43] Exclude columns only filled with NaN values from import --- etrago/cluster/gas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 5b8a07e3..9edf73d0 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -918,7 +918,9 @@ def get_clustering_from_busmap( # import the links and the respective time series with the bus0 and bus1 # values updated from the busmap - io.import_components_from_dataframe(network_gasgrid_c, new_links, "Link") + io.import_components_from_dataframe( + network_gasgrid_c, new_links.loc[:, ~new_links.isna().all()], "Link" + ) if with_time: for attr, df in network.links_t.items(): From 3eeab27b6d27e8ee54336795a7c7de35589d99fe Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:25:22 +0200 Subject: [PATCH 09/43] Exclude columns only filled with NaN values from import of buses --- etrago/cluster/gas.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 9edf73d0..dd2832dd 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -83,10 +83,15 @@ def preprocessing(etrago): # Create network_ch4 (grid nodes in order to create the busmap basis) network_ch4 = Network() - buses_ch4 = etrago.network.buses - links_ch4 = etrago.network.links + buses_ch4 = etrago.network.buses.copy() + links_ch4 = etrago.network.links.copy() + io.import_components_from_dataframe(network_ch4, buses_ch4, "Bus") - io.import_components_from_dataframe(network_ch4, links_ch4, "Link") + network_ch4.madd( + "Link", links_ch4.index, **links_ch4.loc[:, ~links_ch4.isna().any()] + ) + + network_ch4.buses["country"] = buses_ch4.country # Cluster ch4 buses settings = etrago.args["network_clustering"] From 7278ad2ec04691c009f276125c24227a786746f8 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:25:49 +0200 Subject: [PATCH 10/43] Add consitency_check --- etrago/cluster/gas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index dd2832dd..7a560359 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -486,6 +486,7 @@ def gas_postprocessing(etrago, busmap, medoid_idx=None): network_gasgrid_c, etrago.network.carriers, "Carrier" ) + network_gasgrid_c.consistency_check() network_gasgrid_c.determine_network_topology() # Adjust x and y coordinates of 'CH4' and 'H2_grid' medoids From 8f381792acaaa02bf5eb4cb9f5410f9e6e94ee1e Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:26:02 +0200 Subject: [PATCH 11/43] Avoid warnings when setting values with pd.dataframe.at --- etrago/cluster/gas.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 7a560359..287ed093 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -498,32 +498,34 @@ def gas_postprocessing(etrago, busmap, medoid_idx=None): if cluster in busmap[medoid_idx].values: medoid = busmap[medoid_idx][ busmap[medoid_idx] == cluster - ].index + ].index[0] h2_idx = network_gasgrid_c.buses.loc[ (network_gasgrid_c.buses.carrier == "H2_grid") & ( network_gasgrid_c.buses.y - == network_gasgrid_c.buses.at[i, "y"] + == network_gasgrid_c.buses.loc[i, "y"] ) & ( network_gasgrid_c.buses.x - == network_gasgrid_c.buses.at[i, "x"] + == network_gasgrid_c.buses.loc[i, "x"] ) ] if len(h2_idx) > 0: h2_idx = h2_idx.index.tolist()[0] - network_gasgrid_c.buses.at[ + network_gasgrid_c.buses.loc[ h2_idx, "x" ] = etrago.network.buses["x"].loc[medoid] - network_gasgrid_c.buses.at[ + network_gasgrid_c.buses.loc[ h2_idx, "y" ] = etrago.network.buses["y"].loc[medoid] - network_gasgrid_c.buses.at[i, "x"] = etrago.network.buses[ - "x" - ].loc[medoid] - network_gasgrid_c.buses.at[i, "y"] = etrago.network.buses[ - "y" - ].loc[medoid] + + network_gasgrid_c.buses.loc[i, "x"] = etrago.network.buses.loc[ + medoid, "x" + ] + network_gasgrid_c.buses.loc[i, "y"] = etrago.network.buses.loc[ + medoid, "y" + ] + return (network_gasgrid_c, busmap) From 0de7096e1d76861c9a928bd29518a32d16397d18 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:26:23 +0200 Subject: [PATCH 12/43] Apply black --- etrago/cluster/gas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 287ed093..ee3b68e4 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -866,6 +866,7 @@ def get_clustering_from_busmap( A new gas grid pypsa.Network object with aggregated components based on the bus mapping. """ + network_gasgrid_c = Network() # Aggregate buses @@ -936,6 +937,7 @@ def get_clustering_from_busmap( io.import_series_from_dataframe( network_gasgrid_c, df, "Link", attr ) + return network_gasgrid_c From c64b7f5fddc1c64802a7301c8fedd3fd7823206a Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:27:06 +0200 Subject: [PATCH 13/43] Use new strategies for buses and lines --- etrago/cluster/electrical.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 831352da..cf11e2a4 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -43,7 +43,9 @@ group_links, kmean_clustering, kmedoids_dijkstra_clustering, + strategies_buses, strategies_generators, + strategies_lines, strategies_one_ports, ) @@ -891,6 +893,8 @@ def postprocessing(etrago, busmap, busmap_foreign, medoid_idx=None): generator_strategies=strategies_generators(), aggregate_one_ports=aggregate_one_ports, line_length_factor=settings["line_length_factor"], + bus_strategies=strategies_buses(), + line_strategies=strategies_lines(), ) if method == "kmedoids-dijkstra": From 610d40ff41dff3551801591fdeb333b52b4602c7 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:27:38 +0200 Subject: [PATCH 14/43] Avoid warnings when selecting from dataframe --- etrago/cluster/electrical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index cf11e2a4..3b4d6721 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -183,7 +183,7 @@ def adjust_no_electric_network(etrago, busmap, cluster_met): df = df[df["elec"]] if len(df) > 0: - bus_hv = df["bus0"][0] + bus_hv = df.loc[:, "bus0"].iloc[0] if bus_hv == -1: busmap2[bus_ne] = str(bus_ne) From 195bb88c73d600315e07b35d3dd6ba2d4453a269 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:27:59 +0200 Subject: [PATCH 15/43] Add r and v_nom values from transformers to new lines --- etrago/cluster/electrical.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 3b4d6721..26d5b9fb 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -720,6 +720,7 @@ def preprocessing(etrago): [ "bus0", "bus1", + "r", "x", "s_nom", "capital_cost", @@ -732,6 +733,7 @@ def preprocessing(etrago): x=network.transformers.x * (380.0 / transformer_voltages.max(axis=1)) ** 2, length=1, + v_nom = 380., ) .set_index("T" + trafo_index), "Line", From 79eda084f086b3cbd10518a8202e4204fd14f89e Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:28:31 +0200 Subject: [PATCH 16/43] Apply black --- etrago/cluster/electrical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 26d5b9fb..62c0b260 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -733,7 +733,7 @@ def preprocessing(etrago): x=network.transformers.x * (380.0 / transformer_voltages.max(axis=1)) ** 2, length=1, - v_nom = 380., + v_nom=380.0, ) .set_index("T" + trafo_index), "Line", From fae2bdf22d8c0ff1dd1018beb9b249360af0ac55 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:29:01 +0200 Subject: [PATCH 17/43] Choose newest pypsa version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9303239d..cf2bd457 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def read(*names, **kwargs): "loguru", "matplotlib >= 3.0.3", "oedialect", - "pypsa == 0.25.1", + "pypsa == 0.25.2", "rtree", "saio", "scikit-learn", From 43c50c95e4767174144926e4a5d5280bffbe7da1 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:29:40 +0200 Subject: [PATCH 18/43] Temporary solve problems in appl.py Should be removes/replaced later on --- etrago/appl.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/etrago/appl.py b/etrago/appl.py index 083d87b8..641ea11d 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -649,6 +649,14 @@ def run_etrago(args, json_path): # import network from database etrago.build_network_from_db() + etrago.network.lines.loc[etrago.network.lines.r == 0, "r"] = 0.0001 + etrago.network.transformers.loc[ + etrago.network.transformers.r == 0, "r" + ] = 0.0001 + + etrago.network.transformers["v_nom"] = etrago.network.buses.loc[ + etrago.network.transformers.bus0.values, "v_nom" + ].values # adjust network regarding eTraGo setting etrago.adjust_network() @@ -658,6 +666,32 @@ def run_etrago(args, json_path): # spatial clustering etrago.spatial_clustering() + import pandas as pd + + etrago.network.links.min_up_time.fillna(0, inplace=True) + etrago.network.links.min_down_time.fillna(0, inplace=True) + etrago.network.links.up_time_before.fillna(0, inplace=True) + etrago.network.links.down_time_before.fillna(0, inplace=True) + etrago.network.loads_t.p = pd.DataFrame(index=etrago.network.snapshots) + etrago.network.loads_t.q = pd.DataFrame(index=etrago.network.snapshots) + etrago.network.stores_t.p = pd.DataFrame(index=etrago.network.snapshots) + etrago.network.storage_units_t.p = pd.DataFrame( + index=etrago.network.snapshots + ) + etrago.network.stores_t.e = pd.DataFrame(index=etrago.network.snapshots) + etrago.network.stores_t.q = pd.DataFrame(index=etrago.network.snapshots) + etrago.network.stores_t.mu_lower = pd.DataFrame( + index=etrago.network.snapshots + ) + etrago.network.stores_t.mu_upper = pd.DataFrame( + index=etrago.network.snapshots + ) + etrago.network.stores_t.mu_energy_balance = pd.DataFrame( + index=etrago.network.snapshots + ) + etrago.network.storage_units_t.q = pd.DataFrame( + index=etrago.network.snapshots + ) etrago.spatial_clustering_gas() # snapshot clustering From 7ff2ab6195ecfa6cf8e1b9d08725ae578b353974 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:30:42 +0200 Subject: [PATCH 19/43] Remove import of line geometries --- etrago/tools/utilities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index d44171bc..85719e43 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -1059,7 +1059,8 @@ def agg_parallel_lines(l0): .reset_index() .set_index("Line", drop=True) ) - network.lines["geom"] = gpd.GeoSeries.from_wkt(network.lines["geom"]) + + #network.lines["geom"] = gpd.GeoSeries.from_wkt(network.lines["geom"]) return From 14de6b15dc558ec4ef48a3135ca97f42466807be Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:35:21 +0200 Subject: [PATCH 20/43] Black --- etrago/tools/utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 85719e43..8b424c03 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -1059,8 +1059,8 @@ def agg_parallel_lines(l0): .reset_index() .set_index("Line", drop=True) ) - - #network.lines["geom"] = gpd.GeoSeries.from_wkt(network.lines["geom"]) + + # network.lines["geom"] = gpd.GeoSeries.from_wkt(network.lines["geom"]) return From 80e4ce1d91cc21b2b6cba70384fb8a39c4cd0f33 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:43:57 +0200 Subject: [PATCH 21/43] Remove unused imports --- etrago/tools/execute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/etrago/tools/execute.py b/etrago/tools/execute.py index f4ee32fe..ad930c70 100755 --- a/etrago/tools/execute.py +++ b/etrago/tools/execute.py @@ -28,12 +28,10 @@ import time from pypsa.linopf import network_lopf - from pypsa.clustering.spatial import aggregategenerators from pypsa.pf import sub_network_pf import numpy as np import pandas as pd - from etrago.cluster.spatial import strategies_generators from etrago.tools.constraints import Constraints logger = logging.getLogger(__name__) From af9204ee5d1b594a08051e78779416579aa38863 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Oct 2023 15:44:29 +0200 Subject: [PATCH 22/43] Apply isort --- etrago/cluster/gas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index ee3b68e4..fd7916a5 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -26,12 +26,12 @@ if "READTHEDOCS" not in os.environ: import logging - from pypsa.components import Network from pypsa.clustering.spatial import ( aggregatebuses, aggregateoneport, busmap_by_kmeans, ) + from pypsa.components import Network from six import iteritems import numpy as np import pandas as pd From 6e3ab6968c900adaa0760db4e9f793f68b5eeda0 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 22 Nov 2023 09:49:23 +0100 Subject: [PATCH 23/43] Apply isort --- etrago/cluster/spatial.py | 2 +- etrago/tools/constraints.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/etrago/cluster/spatial.py b/etrago/cluster/spatial.py index dcde8ea0..75f69853 100755 --- a/etrago/cluster/spatial.py +++ b/etrago/cluster/spatial.py @@ -30,9 +30,9 @@ from networkx import NetworkXNoPath from pypsa.clustering.spatial import ( - flatten_multiindex, busmap_by_kmeans, busmap_by_stubs, + flatten_multiindex, get_clustering_from_busmap, ) from sklearn.cluster import KMeans diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index e8a61830..b3ae4728 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -21,13 +21,15 @@ """ Constraints.py includes additional constraints for eTraGo-optimizations """ -import os import logging +import os from pyomo.environ import Constraint -from pypsa.descriptors import expand_series +from pypsa.descriptors import ( + expand_series, + get_switchable_as_dense as get_as_dense, +) from pypsa.linopt import define_constraints, define_variables, get_var, linexpr -from pypsa.descriptors import get_switchable_as_dense as get_as_dense import numpy as np import pandas as pd import pyomo.environ as po From 0af37bc8d75e4a5775e4ee272067be50e61f32c9 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 22 Nov 2023 14:17:52 +0100 Subject: [PATCH 24/43] Remove outdated python version 3.8 from github actions --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f2eb32f..fe3de817 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,6 @@ jobs: with: python-version: | 3 - 3.8 3.9 3.10 3.11 From 4d2bee093e0b158c2c4e1d8327d2c1c2c2938926 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 22 Nov 2023 14:26:32 +0100 Subject: [PATCH 25/43] Exclude outdated python version 3.8 from noxfile --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 07366ba1..6aecbb2c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -60,7 +60,7 @@ def flake8(session): session.run("flake8", "--ignore=E722, W605", *cleaned) -@nox.session(python=["3", "3.8", "3.9", "3.10", "3.11"]) +@nox.session(python=["3", "3.9", "3.10", "3.11"]) def build(session): """Build the package and check for packaging errors.""" setdefaults(session) @@ -69,7 +69,7 @@ def build(session): session.run("twine", "check", "dist/eTraGo*") -@nox.session(python=["3", "3.8", "3.9", "3.10", "3.11"]) +@nox.session(python=["3", "3.9", "3.10", "3.11"]) def install(session): """Install the package.""" setdefaults(session) From 994b0a370deb84a061ec5e1f7465b6c77e95e60f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 22 Nov 2023 16:29:10 +0100 Subject: [PATCH 26/43] Remove temporary fixes --- etrago/appl.py | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index bf04fc49..7b29536c 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -687,32 +687,7 @@ def run_etrago(args, json_path): # spatial clustering etrago.spatial_clustering() - import pandas as pd - - etrago.network.links.min_up_time.fillna(0, inplace=True) - etrago.network.links.min_down_time.fillna(0, inplace=True) - etrago.network.links.up_time_before.fillna(0, inplace=True) - etrago.network.links.down_time_before.fillna(0, inplace=True) - etrago.network.loads_t.p = pd.DataFrame(index=etrago.network.snapshots) - etrago.network.loads_t.q = pd.DataFrame(index=etrago.network.snapshots) - etrago.network.stores_t.p = pd.DataFrame(index=etrago.network.snapshots) - etrago.network.storage_units_t.p = pd.DataFrame( - index=etrago.network.snapshots - ) - etrago.network.stores_t.e = pd.DataFrame(index=etrago.network.snapshots) - etrago.network.stores_t.q = pd.DataFrame(index=etrago.network.snapshots) - etrago.network.stores_t.mu_lower = pd.DataFrame( - index=etrago.network.snapshots - ) - etrago.network.stores_t.mu_upper = pd.DataFrame( - index=etrago.network.snapshots - ) - etrago.network.stores_t.mu_energy_balance = pd.DataFrame( - index=etrago.network.snapshots - ) - etrago.network.storage_units_t.q = pd.DataFrame( - index=etrago.network.snapshots - ) + etrago.spatial_clustering_gas() # snapshot clustering From c8b25d9f513849c1346df98a8fc8e44dc0bd591f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 22 Nov 2023 16:29:42 +0100 Subject: [PATCH 27/43] Drop cluster strategies for empty link columns --- etrago/cluster/spatial.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etrago/cluster/spatial.py b/etrago/cluster/spatial.py index 75f69853..f5d46a00 100755 --- a/etrago/cluster/spatial.py +++ b/etrago/cluster/spatial.py @@ -184,10 +184,6 @@ def strategies_links(): "country": nan_links, "build_year": np.mean, "lifetime": np.mean, - "min_up_time": np.mean, - "min_down_time": np.mean, - "up_time_before": np.mean, - "down_time_before": np.mean, "committable": np.all, } From 39bd015a2fd0e2e6f45337737e376bce2a93b84d Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:35:59 +0100 Subject: [PATCH 28/43] Set bus strategie for ehv-clustering --- etrago/cluster/electrical.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 8bf3f57b..7e42f4d3 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -277,6 +277,8 @@ def cluster_on_extra_high_voltage(etrago, busmap, with_time=True): { "x": _leading(busmap, network.buses), "y": _leading(busmap, network.buses), + "geom": lambda x: np.nan, + "country": lambda x: "", }, ) From 734b4160c5e6362fc09d4958c0d9ce73aeb67387 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:36:19 +0100 Subject: [PATCH 29/43] Manually clean network after clustering --- etrago/cluster/electrical.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 7e42f4d3..8050f1ee 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -1077,6 +1077,32 @@ def run_spatial_clustering(self): # which sets slack bus(es). set_control_strategies(self.network) + # Drop nan values after clustering + self.network.links.min_up_time.fillna(0, inplace=True) + self.network.links.min_down_time.fillna(0, inplace=True) + self.network.links.up_time_before.fillna(0, inplace=True) + self.network.links.down_time_before.fillna(0, inplace=True) + self.network.loads_t.p = pd.DataFrame(index=self.network.snapshots) + self.network.loads_t.q = pd.DataFrame(index=self.network.snapshots) + self.network.stores_t.p = pd.DataFrame(index=self.network.snapshots) + self.network.storage_units_t.p = pd.DataFrame( + index=self.network.snapshots + ) + self.network.stores_t.e = pd.DataFrame(index=self.network.snapshots) + self.network.stores_t.q = pd.DataFrame(index=self.network.snapshots) + self.network.stores_t.mu_lower = pd.DataFrame( + index=self.network.snapshots + ) + self.network.stores_t.mu_upper = pd.DataFrame( + index=self.network.snapshots + ) + self.network.stores_t.mu_energy_balance = pd.DataFrame( + index=self.network.snapshots + ) + self.network.storage_units_t.q = pd.DataFrame( + index=self.network.snapshots + ) + logger.info( "Network clustered to {} buses with ".format( self.args["network_clustering"]["n_clusters_AC"] From 28b6fbcf1f6ffac98b22dc0e5cd83bf75db22bc6 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:36:42 +0100 Subject: [PATCH 30/43] Replace deprecated aggregate_generators function --- etrago/cluster/electrical.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 8050f1ee..6f104988 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -331,27 +331,22 @@ def cluster_on_extra_high_voltage(etrago, busmap, with_time=True): io.import_series_from_dataframe(network_c, df, "Link", attr) # dealing with generators - network.generators["weight"] = 1 + # network.generators["weight"] = 1 - new_df, new_pnl = aggregategenerators( - network, busmap, with_time, custom_strategies=strategies_generators() - ) - io.import_components_from_dataframe(network_c, new_df, "Generator") - for attr, df in iteritems(new_pnl): - io.import_series_from_dataframe(network_c, df, "Generator", attr) + for one_port in network.one_port_components.copy(): + if one_port == "Generator": + custom_strategies = strategies_generators() + import pdb - # dealing with all other components - aggregate_one_ports = network.one_port_components.copy() - aggregate_one_ports.discard("Generator") - - for one_port in aggregate_one_ports: - one_port_strategies = strategies_one_ports() + pdb.set_trace() + else: + custom_strategies = strategies_one_ports().get(one_port, {}) new_df, new_pnl = aggregateoneport( network, busmap, component=one_port, with_time=with_time, - custom_strategies=one_port_strategies.get(one_port, {}), + custom_strategies=custom_strategies, ) io.import_components_from_dataframe(network_c, new_df, one_port) for attr, df in iteritems(new_pnl): From d34c3f3ac92bd3944770f214ae1d9a28d23455fa Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:38:30 +0100 Subject: [PATCH 31/43] Allow pf post lopf when using pypsa>0.25 --- etrago/tools/execute.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/etrago/tools/execute.py b/etrago/tools/execute.py index 0161fc10..509b674d 100755 --- a/etrago/tools/execute.py +++ b/etrago/tools/execute.py @@ -553,20 +553,29 @@ def import_gen_from_links(network, drop_small_capacities=True): df["efficiency"] = gas_to_add.groupby(["bus", "carrier"]).efficiency.mean() df["control"] = "PV" df.reset_index(inplace=True) + df.index = df.bus + " " + df.carrier # Aggregate disptach time series for new generators gas_to_add["bus1_carrier"] = gas_to_add.bus + " " + gas_to_add.carrier - df_t = ( - network.links_t.p1[gas_to_add.index] - .groupby(gas_to_add.bus1_carrier, axis=1) - .sum() - * -1 - ) + + if not network.links_t.p1.empty: + df_t = ( + network.links_t.p1[gas_to_add.index] + .groupby(gas_to_add.bus1_carrier, axis=1) + .sum() + * -1 + ) # Insert aggregated generators their dispatch time series network.madd("Generator", df.index, **df) - network.import_series_from_dataframe(df_t, "Generator", "p") + if not network.links_t.p1.empty: + network.import_series_from_dataframe(df_t, "Generator", "p") + network.import_series_from_dataframe( + pd.DataFrame(index=df_t.index, columns=df_t.columns, data=1.0), + "Generator", + "status", + ) # Drop links now modelled as generator network.mremove("Link", gas_to_add.index) @@ -677,7 +686,10 @@ def drop_foreign_components(network): for comp in sorted(foreign_series): attr = sorted(foreign_series[comp]) for a in attr: - if not foreign_series[comp][a].empty: + if ( + not foreign_series[comp][a].empty + and not (foreign_series[comp][a] == 0.0).all().all() + ): if a != "p_max_pu": if a in ["q_set", "e_max_pu", "e_min_pu"]: g_in_q_set = foreign_comp[comp][ From b342475dd59e4ad96d1ec736bd4a73a473599bec Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:38:51 +0100 Subject: [PATCH 32/43] Black --- etrago/tools/execute.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etrago/tools/execute.py b/etrago/tools/execute.py index 509b674d..613a0a1e 100755 --- a/etrago/tools/execute.py +++ b/etrago/tools/execute.py @@ -423,6 +423,7 @@ def dispatch_disaggregation(self): # define number of slices and corresponding slice length no_slices = self.args["temporal_disaggregation"]["no_slices"] slice_len = int(len(self.network.snapshots) / no_slices) + # transition snapshots defining start and end of slices transits = self.network.snapshots[0::slice_len] if len(transits) > 1: From 721dd04bb0fb322266037cbd41df95a92e784b89 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:39:16 +0100 Subject: [PATCH 33/43] Extended cluster strategies for links according to new pypsa columns --- etrago/cluster/spatial.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etrago/cluster/spatial.py b/etrago/cluster/spatial.py index f5d46a00..75f69853 100755 --- a/etrago/cluster/spatial.py +++ b/etrago/cluster/spatial.py @@ -184,6 +184,10 @@ def strategies_links(): "country": nan_links, "build_year": np.mean, "lifetime": np.mean, + "min_up_time": np.mean, + "min_down_time": np.mean, + "up_time_before": np.mean, + "down_time_before": np.mean, "committable": np.all, } From 96c916ca8ac57a013e6d518d57b3db11eb380653 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 24 Nov 2023 12:44:12 +0100 Subject: [PATCH 34/43] Delete unused import --- etrago/cluster/electrical.py | 1 - 1 file changed, 1 deletion(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 6f104988..13094d6b 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -29,7 +29,6 @@ from pypsa import Network from pypsa.clustering.spatial import ( aggregatebuses, - aggregategenerators, aggregateoneport, get_clustering_from_busmap, ) From 922e6421bab3038caf3f85532bbf69b418394dd4 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 30 Nov 2023 09:56:24 +0100 Subject: [PATCH 35/43] Delete pdb statement --- etrago/cluster/electrical.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 13094d6b..a1564e56 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -335,9 +335,7 @@ def cluster_on_extra_high_voltage(etrago, busmap, with_time=True): for one_port in network.one_port_components.copy(): if one_port == "Generator": custom_strategies = strategies_generators() - import pdb - pdb.set_trace() else: custom_strategies = strategies_one_ports().get(one_port, {}) new_df, new_pnl = aggregateoneport( From f55fa3ecfb451fb8ddf4c2f8951d95559ae7125f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 30 Nov 2023 09:56:42 +0100 Subject: [PATCH 36/43] Replace filled output timeseries with empty dataframe Within pypsa's network clustering, all output timeseries are filled with 0.0. This results into problems in the following clustering steps. To avoid that, the output timeseries data is replaced with empty dataframes. --- etrago/cluster/electrical.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index a1564e56..217b3826 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -450,6 +450,13 @@ def ehv_clustering(self): self.update_busmap(busmap) self.buses_by_country() + # Drop nan values in timeseries after clustering + for c in self.network.iterate_components(): + for pnl in c.attrs[ + (c.attrs.status == "Output") & (c.attrs.varying) + ].index: + c.pnl[pnl] = pd.DataFrame(index=self.network.snapshots) + logger.info("Network clustered to EHV-grid") @@ -1074,26 +1081,12 @@ def run_spatial_clustering(self): self.network.links.min_down_time.fillna(0, inplace=True) self.network.links.up_time_before.fillna(0, inplace=True) self.network.links.down_time_before.fillna(0, inplace=True) - self.network.loads_t.p = pd.DataFrame(index=self.network.snapshots) - self.network.loads_t.q = pd.DataFrame(index=self.network.snapshots) - self.network.stores_t.p = pd.DataFrame(index=self.network.snapshots) - self.network.storage_units_t.p = pd.DataFrame( - index=self.network.snapshots - ) - self.network.stores_t.e = pd.DataFrame(index=self.network.snapshots) - self.network.stores_t.q = pd.DataFrame(index=self.network.snapshots) - self.network.stores_t.mu_lower = pd.DataFrame( - index=self.network.snapshots - ) - self.network.stores_t.mu_upper = pd.DataFrame( - index=self.network.snapshots - ) - self.network.stores_t.mu_energy_balance = pd.DataFrame( - index=self.network.snapshots - ) - self.network.storage_units_t.q = pd.DataFrame( - index=self.network.snapshots - ) + # Drop nan values in timeseries after clustering + for c in self.network.iterate_components(): + for pnl in c.attrs[ + (c.attrs.status == "Output") & (c.attrs.varying) + ].index: + c.pnl[pnl] = pd.DataFrame(index=self.network.snapshots) logger.info( "Network clustered to {} buses with ".format( From 77e1850f51325a9ebb36bc25a0307a9ddf39b763 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 30 Nov 2023 09:58:50 +0100 Subject: [PATCH 37/43] Only use p_max_pu timeseries data for generators that have this data PyPSA's networkclustering now deletes timeseries that only inlcude the default value. When running smaller example calculations eith only a few snapshots, it could happen, that the p_max_pu time series of some wind or solar generator is always 1.0 To avoid key errors in these cases, it was required to not use the carrier to decide if a generator has a p_max_pu time series but directly look at the existing columns. --- etrago/cluster/electrical.py | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index 217b3826..b80b561d 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -935,41 +935,12 @@ def calc_availability_factor(gen): cannot be found in the dictionary, it is assumed to be 1. """ - - if gen["carrier"] in time_dependent: + if gen.name in network.generators_t.p_max_pu.columns: cf = network.generators_t["p_max_pu"].loc[:, gen.name].mean() else: - try: - cf = fixed_capacity_fac[gen["carrier"]] - except KeyError: - cf = 1 - return cf + cf = network.generators.loc[gen.name, "p_max_pu"] - time_dependent = [ - "solar_rooftop", - "solar", - "wind_onshore", - "wind_offshore", - ] - fixed_capacity_fac = { - # A value of 1 is given to power plants where its availability - # does not depend on the weather - "industrial_gas_CHP": 1, - "industrial_biomass_CHP": 1, - "biomass": 1, - "central_biomass_CHP": 1, - "central_gas_CHP": 1, - "OCGT": 1, - "other_non_renewable": 1, - "run_of_river": 0.50, - "reservoir": 1, - "gas": 1, - "oil": 1, - "others": 1, - "coal": 1, - "lignite": 1, - "nuclear": 1, - } + return cf gen = network.generators[network.generators.carrier != "load shedding"][ ["bus", "carrier", "p_nom"] From 234a7d65d80cd6e3e637f49d8801c021815c4dca Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 30 Nov 2023 10:02:54 +0100 Subject: [PATCH 38/43] Avoid pyomo deprecation warnings --- etrago/tools/constraints.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index b3ae4728..de794c41 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -1820,8 +1820,7 @@ def inter_storage_soc_rule(m, s, i): L. Kotzur et al: 'Time series aggregation for energy system design: Modeling seasonal storage', 2018, equation no. 19 """ - - if i == network.model.candidates[-1]: + if i == network.model.candidates.at(-1): last_hour = network.cluster["last_hour_RepresentativeDay"][i] expr = po.Constraint.Skip else: @@ -1849,7 +1848,7 @@ def inter_storage_soc_rule(m, s, i): return expr def inter_store_soc_rule(m, s, i): - if i == network.model.candidates[-1]: + if i == network.model.candidates.at(-1): last_hour = network.cluster["last_hour_RepresentativeDay"][i] expr = po.Constraint.Skip else: From 3e33c47cb7bf924bdc2949719f2f61216e7a84dc Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 30 Nov 2023 10:03:39 +0100 Subject: [PATCH 39/43] Make sure that the last snapshot of the aggreagted timeseries is included Before it was checked if the last snashot of the disaggreagted timeseries is included which led to duplicated index values. --- etrago/tools/execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/tools/execute.py b/etrago/tools/execute.py index 613a0a1e..78849440 100755 --- a/etrago/tools/execute.py +++ b/etrago/tools/execute.py @@ -428,7 +428,7 @@ def dispatch_disaggregation(self): transits = self.network.snapshots[0::slice_len] if len(transits) > 1: transits = transits[1:] - if transits[-1] != self.network_tsa.snapshots[-1]: + if transits[-1] != self.network.snapshots[-1]: transits = transits.insert( (len(transits)), self.network.snapshots[-1] ) From 79bfa54a86a2f6c8ff4c07df169f0a8cb1d245f7 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 1 Dec 2023 09:39:08 +0100 Subject: [PATCH 40/43] Move manual fixes into function --- etrago/tools/utilities.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 7a3dfdfb..51ee2019 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -2858,3 +2858,14 @@ def manual_fixes_datamodel(etrago): etrago.network.links.loc[ etrago.network.links.carrier == "H2_to_CH4", "marginal_cost" ] = 25 + + # Set r value if missing + etrago.network.lines.loc[etrago.network.lines.r == 0, "r"] = 0.0001 + etrago.network.transformers.loc[ + etrago.network.transformers.r == 0, "r" + ] = 0.0001 + + # Set vnom of transformers + etrago.network.transformers["v_nom"] = etrago.network.buses.loc[ + etrago.network.transformers.bus0.values, "v_nom" + ].values From 9daf55f527746a573f62b6e2dd071a4ae04af566 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 1 Dec 2023 10:01:35 +0100 Subject: [PATCH 41/43] Remove manual fixes from appl.py --- etrago/appl.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 7b29536c..ee04e437 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -670,14 +670,6 @@ def run_etrago(args, json_path): # import network from database etrago.build_network_from_db() - etrago.network.lines.loc[etrago.network.lines.r == 0, "r"] = 0.0001 - etrago.network.transformers.loc[ - etrago.network.transformers.r == 0, "r" - ] = 0.0001 - - etrago.network.transformers["v_nom"] = etrago.network.buses.loc[ - etrago.network.transformers.bus0.values, "v_nom" - ].values # adjust network regarding eTraGo setting etrago.adjust_network() From 25de9e92576bdd7d8fde42d6087b8c5d8bd884d0 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 1 Dec 2023 10:12:55 +0100 Subject: [PATCH 42/43] Add print statement when resitances are set automatically --- etrago/tools/utilities.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 51ee2019..e141e7af 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -2860,7 +2860,28 @@ def manual_fixes_datamodel(etrago): ] = 25 # Set r value if missing + if not etrago.network.lines.loc[etrago.network.lines.r == 0, "r"].empty: + logger.info( + f""" + There are {len( + etrago.network.lines.loc[etrago.network.lines.r == 0, "r"] + )} lines without a resistance (r) in the data model. + The resistance of these lines will be automatically set to 0.0001. + """ + ) + etrago.network.lines.loc[etrago.network.lines.r == 0, "r"] = 0.0001 + + if not etrago.network.transformers.loc[ + etrago.network.transformers.r == 0, "r" + ].empty: + logger.info( + f"""There are {len(etrago.network.transformers.loc[ + etrago.network.transformers.r == 0, "r"] + )} trafos without a resistance (r) in the data model. + The resistance of these trafos will be automatically set to 0.0001. + """ + ) etrago.network.transformers.loc[ etrago.network.transformers.r == 0, "r" ] = 0.0001 From 530d7538e96e361a0d0d9460bce83b4724e57e93 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Fri, 1 Dec 2023 10:32:23 +0100 Subject: [PATCH 43/43] Remove nan values within postprocessing --- etrago/cluster/electrical.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/etrago/cluster/electrical.py b/etrago/cluster/electrical.py index b80b561d..081fe543 100755 --- a/etrago/cluster/electrical.py +++ b/etrago/cluster/electrical.py @@ -866,6 +866,18 @@ def postprocessing(etrago, busmap, busmap_foreign, medoid_idx=None): line_strategies=strategies_lines(), ) + # Drop nan values after clustering + clustering.network.links.min_up_time.fillna(0, inplace=True) + clustering.network.links.min_down_time.fillna(0, inplace=True) + clustering.network.links.up_time_before.fillna(0, inplace=True) + clustering.network.links.down_time_before.fillna(0, inplace=True) + # Drop nan values in timeseries after clustering + for c in clustering.network.iterate_components(): + for pnl in c.attrs[ + (c.attrs.status == "Output") & (c.attrs.varying) + ].index: + c.pnl[pnl] = pd.DataFrame(index=clustering.network.snapshots) + if method == "kmedoids-dijkstra": for i in clustering.network.buses[ clustering.network.buses.carrier == "AC" @@ -1047,18 +1059,6 @@ def run_spatial_clustering(self): # which sets slack bus(es). set_control_strategies(self.network) - # Drop nan values after clustering - self.network.links.min_up_time.fillna(0, inplace=True) - self.network.links.min_down_time.fillna(0, inplace=True) - self.network.links.up_time_before.fillna(0, inplace=True) - self.network.links.down_time_before.fillna(0, inplace=True) - # Drop nan values in timeseries after clustering - for c in self.network.iterate_components(): - for pnl in c.attrs[ - (c.attrs.status == "Output") & (c.attrs.varying) - ].index: - c.pnl[pnl] = pd.DataFrame(index=self.network.snapshots) - logger.info( "Network clustered to {} buses with ".format( self.args["network_clustering"]["n_clusters_AC"]