Fixes/#660 fix ehv clustering

etrago/appl.py

@@ -102,7 +102,11 @@
     "generator_noise": 789456,  # apply generator noise, False or seed number
     "extra_functionality": {},  # Choose function name or {}
     # Spatial Complexity:
-    "network_clustering_ehv": False,  # clustering of HV buses to EHV buses
+    "delete_dispensable_ac_buses": True, # bool. Find and delete expendable buses
+    "network_clustering_ehv": {
+        "active": False,  # choose if clustering of HV buses to EHV buses is activated
+        "busmap": False, # False or path to stored busmap
+    },
     "network_clustering": {
         "active": True,  # choose if clustering is activated
         "method": "kmedoids-dijkstra",  # choose clustering method: kmeans or kmedoids-dijkstra
@@ -383,11 +387,28 @@ def run_etrago(args, json_path):
             Limit overall energy production country-wise for each generator
             by carrier. Set upper/lower limit in p.u.
 
-    network_clustering_ehv : bool
+    delete_dispensable_ac_buses: bool
+        Choose if electrical buses that are only connecting two lines should be
+        removed. These buses have no other components attached to them. The
+        connected lines are merged. This reduces the spatial complexity without
+        losing any accuracy.
+        Default: True.
+    network_clustering_ehv : dict
+        Choose if you want to apply an extra high voltage clustering to the
+        electrical network.
+        The provided dictionary can have the following entries:
+
+        * "active" : bool
         Choose if you want to cluster the full HV/EHV dataset down to only the
         EHV buses. In that case, all HV buses are assigned to their closest EHV
         substation, taking into account the shortest distance on power lines.
         Default: False.
+        * "busmap" : str
+        Choose if an stored busmap can be used to make the process quicker, or
+        a new busmap must be calculated. False or path to the busmap in csv
+        format should be given.
+        Default: False
+
     network_clustering : dict
         Choose if you want to apply a clustering of all network buses and
         specify settings.

etrago/args.json

@@ -51,7 +51,11 @@
     },
     "generator_noise": 789456,
     "extra_functionality": {},
-    "network_clustering_ehv": false,
+    "delete_dispensable_ac_buses": true,
+    "network_clustering_ehv": {
+        "active": false,
+        "busmap": false,
+    },
     "network_clustering": {
         "active": true,
         "method": "kmedoids-dijkstra",

etrago/cluster/electrical.py

@@ -290,11 +290,6 @@ def cluster_on_extra_high_voltage(etrago, busmap, with_time=True):
         etrago, busmap, cluster_met="ehv"
     )
 
-    pd.DataFrame(busmap.items(), columns=["bus0", "bus1"]).to_csv(
-        "ehv_elecgrid_busmap_result.csv",
-        index=False,
-    )
-
     buses = aggregatebuses(
         network,
         busmap,
@@ -321,6 +316,8 @@ def cluster_on_extra_high_voltage(etrago, busmap, with_time=True):
     # Dealing with links
     links = network.links.copy()
     dc_links = links[links["carrier"] == "DC"]
+    # Discard links connected to buses under 220 kV
+    dc_links = dc_links[dc_links.bus0.isin(buses.index)]
     links = links[links["carrier"] != "DC"]
 
     new_links = (
@@ -379,7 +376,6 @@ def cluster_on_extra_high_voltage(etrago, busmap, with_time=True):
             io.import_series_from_dataframe(network_c, df, one_port, attr)
 
     network_c.links, network_c.links_t = group_links(network_c)
-
     network_c.determine_network_topology()
 
     return (network_c, busmap)
@@ -412,10 +408,10 @@ def delete_ehv_buses_no_lines(network):
     buses_ac["with_gen"] = buses_ac.index.isin(network.generators.bus)
 
     delete_buses = buses_ac[
-        (buses_ac["with_line"] is False)
-        & (buses_ac["with_load"] is False)
-        & (buses_ac["with_link"] is False)
-        & (buses_ac["with_gen"] is False)
+        (buses_ac["with_line"] == False)
+        & (buses_ac["with_load"] == False)
+        & (buses_ac["with_link"] == False)
+        & (buses_ac["with_gen"] == False)
     ].index
 
     if len(delete_buses):
@@ -453,8 +449,8 @@ def ehv_clustering(self):
     """
     Cluster the network based on Extra High Voltage (EHV) grid.
 
-    If `network_clustering_ehv` argument is True, the function clusters the
-    network based on the EHV grid.
+    If 'active' in the `network_clustering_ehv` argument is True, the function
+    clusters the network based on the EHV grid.
 
     Parameters
     ----------
@@ -466,7 +462,7 @@ def ehv_clustering(self):
     None
     """
 
-    if self.args["network_clustering_ehv"]:
+    if self.args["network_clustering_ehv"]["active"]:
         logger.info("Start ehv clustering")
 
         self.network.generators.control = "PV"

etrago/cluster/spatial.py

@@ -362,7 +362,7 @@ def shortest_path(paths, graph):
     return df
 
 
-def busmap_by_shortest_path(etrago, scn_name, fromlvl, tolvl, cpu_cores=4):
+def busmap_by_shortest_path(etrago, fromlvl, tolvl, cpu_cores=4):
     """
     Creates a busmap for the EHV-Clustering between voltage levels based
     on dijkstra shortest path. The result is automatically written to the
@@ -379,8 +379,6 @@ def busmap_by_shortest_path(etrago, scn_name, fromlvl, tolvl, cpu_cores=4):
         Container for all network components.
     session : sqlalchemy.orm.session.Session object
         Establishes interactions with the database.
-    scn_name : str
-        Name of the scenario.
     fromlvl : list
         List of voltage-levels to cluster.
     tolvl : list
@@ -393,14 +391,18 @@ def busmap_by_shortest_path(etrago, scn_name, fromlvl, tolvl, cpu_cores=4):
     None
     """
 
-    # cpu_cores = mp.cpu_count()
-
     # data preperation
     s_buses = buses_grid_linked(etrago.network, fromlvl)
     lines = connected_grid_lines(etrago.network, s_buses)
     transformer = connected_transformer(etrago.network, s_buses)
     mask = transformer.bus1.isin(buses_of_vlvl(etrago.network, tolvl))
 
+    dc = etrago.network.links[etrago.network.links.carrier == "DC"]
+    dc.index = "DC_" + dc.index
+    lines_plus_dc = pd.concat([lines, dc])
+    lines_plus_dc = lines_plus_dc[etrago.network.lines.columns]
+    lines_plus_dc["carrier"] = "AC"
+
     # temporary end points, later replaced by bus1 pendant
     t_buses = transformer[mask].bus0
 
@@ -409,7 +411,8 @@ def busmap_by_shortest_path(etrago, scn_name, fromlvl, tolvl, cpu_cores=4):
 
     # graph creation
     edges = [
-        (row.bus0, row.bus1, row.length, ix) for ix, row in lines.iterrows()
+        (row.bus0, row.bus1, row.length, ix)
+        for ix, row in lines_plus_dc.iterrows()
     ]
     M = graph_from_edges(edges)
 
@@ -469,25 +472,11 @@ def busmap_by_shortest_path(etrago, scn_name, fromlvl, tolvl, cpu_cores=4):
     df = pd.concat([df, tofill], ignore_index=True, axis=0)
     df.drop_duplicates(inplace=True)
 
-    # prepare data for export
-
-    df["scn_name"] = scn_name
-    df["version"] = etrago.args["gridversion"]
-
-    if not df.version.any():
-        df.version = "testcase"
-
     df.rename(columns={"source": "bus0", "target": "bus1"}, inplace=True)
-    df.set_index(["scn_name", "bus0", "bus1"], inplace=True)
 
-    df.to_sql(
-        "egon_etrago_hv_busmap",
-        con=etrago.engine,
-        schema="grid",
-        if_exists="append",
-    )
+    busmap = pd.Series(df.bus1.values, index=df.bus0).to_dict()
 
-    return
+    return busmap
 
 
 def busmap_from_psql(etrago):
@@ -510,52 +499,26 @@ def busmap_from_psql(etrago):
     busmap : dict
         Maps old bus_ids to new bus_ids.
     """
-    scn_name = (
-        etrago.args["scn_name"]
-        if etrago.args["scn_extension"] is None
-        else etrago.args["scn_name"]
-        + "_ext_"
-        + "_".join(etrago.args["scn_extension"])
-    )
-
-    from saio.grid import egon_etrago_hv_busmap
-
-    filter_version = etrago.args["gridversion"]
-
-    if not filter_version:
-        filter_version = "testcase"
-
-    def fetch():
-        query = (
-            etrago.session.query(
-                egon_etrago_hv_busmap.bus0, egon_etrago_hv_busmap.bus1
-            )
-            .filter(egon_etrago_hv_busmap.scn_name == scn_name)
-            .filter(egon_etrago_hv_busmap.version == filter_version)
-        )
-
-        return dict(query.all())
-
-    busmap = fetch()
-
-    # TODO: Or better try/except/finally
-    if not busmap:
-        print("Busmap does not exist and will be created.\n")
 
+    if etrago.args["network_clustering_ehv"]["busmap"] is False:
         cpu_cores = etrago.args["network_clustering"]["CPU_cores"]
         if cpu_cores == "max":
             cpu_cores = mp.cpu_count()
         else:
             cpu_cores = int(cpu_cores)
 
-        busmap_by_shortest_path(
+        busmap = busmap_by_shortest_path(
             etrago,
-            scn_name,
             fromlvl=[110],
             tolvl=[220, 380, 400, 450],
             cpu_cores=cpu_cores,
         )
-        busmap = fetch()
+        pd.DataFrame(busmap.items(), columns=["bus0", "bus1"]).to_csv(
+            "ehv_elecgrid_busmap_result.csv",
+            index=False,
+        )
+    else:
+        busmap = pd.read_csv(etrago.args["network_clustering_ehv"]["busmap"])
 
     return busmap
 

etrago/tools/utilities.py

@@ -127,6 +127,16 @@ def buses_grid_linked(network, voltage_level):
     mask = (
         network.buses.index.isin(network.lines.bus0)
         | (network.buses.index.isin(network.lines.bus1))
+        | (
+            network.buses.index.isin(
+                network.links.loc[network.links.carrier == "DC", "bus0"]
+            )
+        )
+        | (
+            network.buses.index.isin(
+                network.links.loc[network.links.carrier == "DC", "bus1"]
+            )
+        )
     ) & (network.buses.v_nom.isin(voltage_level))
 
     df = network.buses[mask]
@@ -1079,6 +1089,8 @@ def delete_dispensable_ac_buses(etrago):
     None.
 
     """
+    if etrago.args["delete_dispensable_ac_buses"] is False:
+        return
 
     def delete_buses(delete_buses, network):
         drop_buses = delete_buses.index.to_list()
@@ -1866,7 +1878,7 @@ def get_clustering_data(self, path):
 
     """
 
-    if (self.args["network_clustering_ehv"]) | (
+    if (self.args["network_clustering_ehv"]["active"]) | (
         self.args["network_clustering"]["active"]
     ):
         path_clus = os.path.join(path, "clustering")