From eadbc968f86e4230a0c89958a39bbe01ae613598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 2 Feb 2023 06:25:14 +0100 Subject: [PATCH 01/49] Revert reverting package installation during CI This reverts commit 790afef6238f9450744b9347b1264c1296615a82 so that the code can be merged back into dev, once issue #589 is fixed. --- .github/workflows/ci.yaml | 4 ++++ noxfile.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 83781f12e..4f2eb32f3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,3 +52,7 @@ jobs: run: "python -m nox -s build" env: PLATFORM: ${{ matrix.os.image }} + - name: "Install the package" + run: "python -m nox -s install" + env: + PLATFORM: ${{ matrix.os.image }} diff --git a/noxfile.py b/noxfile.py index 98f0c9eb6..7c12709ae 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,3 +1,4 @@ +from pathlib import Path from pprint import pformat import nox @@ -58,3 +59,11 @@ def build(session): session.install("twine") session.run("python", "setup.py", "bdist", "bdist_wheel") session.run("twine", "check", "dist/eTraGo*") + + +@nox.session(python=["3", "3.8", "3.9", "3.10", "3.11"]) +def install(session): + """Install the package.""" + setdefaults(session) + session.run("python", "-mpip", "install", "--upgrade", "pip") + session.run("python", "-mpip", "install", *Path("dist").glob("*.whl")) From dc8a7ed2469cafd84514d0f040d77f0d7673214e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Sat, 4 Feb 2023 00:56:57 +0100 Subject: [PATCH 02/49] Force an error on "sklearn" installation attempts According to "sklearn"'s [description][0], installing the package doesn't always raise an error, but only according to a specific schedule. Obviously this doesn't work for tests. Fortunately setting the `SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL` environment variable to `False` always forces the error. [0]: https://pypi.org/project/sklearn/#description --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index 7c12709ae..9764d83de 100644 --- a/noxfile.py +++ b/noxfile.py @@ -65,5 +65,6 @@ def build(session): def install(session): """Install the package.""" setdefaults(session) + session.env["SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL"] = "False" session.run("python", "-mpip", "install", "--upgrade", "pip") session.run("python", "-mpip", "install", *Path("dist").glob("*.whl")) From 0a17cac0b07f24030a289f11063537e2be041127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 22 Mar 2023 19:08:27 +0100 Subject: [PATCH 03/49] Call `UniformDisaggregation` with named arguments Doesn't change the behaviour of the code but makes it easier to understand. --- etrago/cluster/disaggregation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 0b4378836..b7531d551 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -733,9 +733,9 @@ def run_disaggregation(self): ) elif disagg == "uniform": disaggregation = UniformDisaggregation( - self.disaggregated_network, - self.network, - self.clustering, + original_network=self.disaggregated_network, + clustered_network=self.network, + clustering=self.clustering, skip=skip, ) From db8232c1b4aca4dc60794860dc4fbaaa79f21dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 22 Mar 2023 12:16:36 +0100 Subject: [PATCH 04/49] Replace `print`s with proper logging Use `loguru` for the logging. Get rid of the `"---"` lines in logging, because log messages should contain some explicit information. --- etrago/cluster/disaggregation.py | 37 ++++++++++++++++---------------- setup.py | 1 + 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index b7531d551..0371de296 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -4,6 +4,7 @@ import cProfile import time +from loguru import logger as log from pyomo.environ import Constraint from pypsa import Network import pandas as pd @@ -275,8 +276,7 @@ def solve(self, scenario, solver): } profile = cProfile.Profile() for i, cluster in enumerate(sorted(clusters)): - print("---") - print("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n)) + log.info("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n)) profile.enable() t = time.time() partial_network, externals = self.construct_partial_network( @@ -284,9 +284,9 @@ def solve(self, scenario, solver): ) profile.disable() self.stats["clusters"].loc[cluster, "decompose"] = time.time() - t - print( - "Decomposed in ", - self.stats["clusters"].loc[cluster, "decompose"], + log.info( + "Decomposed in " + f'{self.stats["clusters"].loc[cluster, "decompose"]}' ) t = time.time() profile.enable() @@ -295,32 +295,33 @@ def solve(self, scenario, solver): ) profile.disable() self.stats["clusters"].loc[cluster, "spread"] = time.time() - t - print( - "Result distributed in ", - self.stats["clusters"].loc[cluster, "spread"], + log.info( + "Result distributed in " + f'{self.stats["clusters"].loc[cluster, "spread"]}' ) profile.enable() t = time.time() self.transfer_results(partial_network, externals) profile.disable() self.stats["clusters"].loc[cluster, "transfer"] = time.time() - t - print( - "Results transferred in ", - self.stats["clusters"].loc[cluster, "transfer"], + log.info( + "Results transferred in " + f'{self.stats["clusters"].loc[cluster, "transfer"]}' ) profile.enable() t = time.time() - print("---") fs = (mc("sum"), mc("sum")) for bt, ts in ( ("generators", {"p": fs, "q": fs}), ("storage_units", {"p": fs, "state_of_charge": fs, "q": fs}), ): - print("Attribute sums, {}, clustered - disaggregated:".format(bt)) + log.info( + "Attribute sums, {}, clustered - disaggregated:".format(bt) + ) cnb = getattr(self.clustered_network, bt) onb = getattr(self.original_network, bt) - print( + log.info( "{:>{}}: {}".format( "p_nom_opt", 4 + len("state_of_charge"), @@ -329,11 +330,11 @@ def solve(self, scenario, solver): ) ) - print("Series sums, {}, clustered - disaggregated:".format(bt)) + log.info("Series sums, {}, clustered - disaggregated:".format(bt)) cnb = getattr(self.clustered_network, bt + "_t") onb = getattr(self.original_network, bt + "_t") for s in ts: - print( + log.info( "{:>{}}: {}".format( s, 4 + len("state_of_charge"), @@ -343,7 +344,7 @@ def solve(self, scenario, solver): ) profile.disable() self.stats["check"] = time.time() - t - print("Checks computed in ", self.stats["check"]) + log.info("Checks computed in {self.stats['check']}") # profile.print_stats(sort='cumtime') @@ -749,7 +750,7 @@ def run_disaggregation(self): self.disaggregated_network.generators_t.q.fillna(0, inplace=True) self.disaggregated_network.results = self.network.results - print( + log.info( "Time for overall desaggregation [min]: {:.2}".format( (time.time() - t) / 60 ) diff --git a/setup.py b/setup.py index 4ea9b7ab6..c9fd3aaa1 100755 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ def read(*names, **kwargs): "egoio == 0.4.7", "geoalchemy2 >= 0.3.0", "geopandas", + "loguru", "matplotlib >= 3.0.3", "oedialect", # PyPSA uses a deprecated import that errors with Pyomo 6.4.3. From 4716abd94daa5b39759ec9088896893aafd05798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 22 Mar 2023 18:44:14 +0100 Subject: [PATCH 05/49] Log more stuff --- etrago/cluster/disaggregation.py | 3 +++ etrago/tools/utilities.py | 1 + 2 files changed, 4 insertions(+) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 0371de296..e9281cedc 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -520,6 +520,7 @@ class UniformDisaggregation(Disaggregation): def solve_partial_network( self, cluster, partial_network, scenario, solver=None ): + log.debug("Solving partial network.") bustypes = { "generators": {"group_by": ("carrier",), "series": ("p", "q")}, "storage_units": { @@ -541,6 +542,7 @@ def solve_partial_network( } filters = {"q": lambda o: o.control == "PV"} for bustype in bustypes: + log.debug(f"Decomposing {bustype}.") pn_t = getattr(partial_network, bustype + "_t") cl_t = getattr(self.clustered_network, bustype + "_t") pn_buses = getattr(partial_network, bustype) @@ -720,6 +722,7 @@ def update_constraints(network, externals): def run_disaggregation(self): + log.debug("Running disaggregation.") if self.clustering: disagg = self.args.get("disaggregation") skip = () if self.args["pf_post_lopf"]["active"] else ("q",) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index fddaab0eb..1524e19d8 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -571,6 +571,7 @@ def load_shedding(self, temporal_disaggregation=False, **kwargs): ------- """ + logger.debug("Shedding the load.") if self.args["load_shedding"]: if temporal_disaggregation: network = self.network_tsa From d06926b08452903b32885cf4d9e66d3561e06c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 23 Mar 2023 18:13:04 +0100 Subject: [PATCH 06/49] Make local variable `busflags` an argument When closing over local variables, changing the outer variable also changes the value inside the function. Making the local variable an argument with default value prevents that and it's also a tiny bit more efficient. --- etrago/cluster/disaggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index e9281cedc..58deb414d 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -77,7 +77,7 @@ def construct_partial_network(self, cluster, scenario): # find all lines that have at least one bus inside the cluster busflags = self.buses["cluster"] == cluster - def is_bus_in_cluster(conn): + def is_bus_in_cluster(conn, busflags=busflags): return busflags[conn] # Copy configurations to new network From a8c24ee3164d89dd6bb8771ca060f0160a30a7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 23 Mar 2023 18:31:27 +0100 Subject: [PATCH 07/49] Factor `rows` out into a local variable And put a type annotation on the local variable. Helps me to figure out what values are getting passed around, i.e. it makes the code more readable for me. Also, actually makes the code shorter. --- etrago/cluster/disaggregation.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 58deb414d..b052ba45b 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -93,14 +93,12 @@ def is_bus_in_cluster(conn, busflags=busflags): line_types = ["lines", "links", "transformers"] for line_type in line_types: + rows: pd.DataFrame = getattr(self.original_network, line_type) # Copy all lines that reside entirely inside the cluster ... setattr( partial_network, line_type, - filter_internal_connector( - getattr(self.original_network, line_type), - is_bus_in_cluster, - ), + filter_internal_connector(rows, is_bus_in_cluster), ) # ... and their time series @@ -115,7 +113,7 @@ def is_bus_in_cluster(conn, busflags=busflags): # Copy all lines whose `bus0` lies within the cluster left_external_connectors = filter_left_external_connector( - getattr(self.original_network, line_type), is_bus_in_cluster + rows, is_bus_in_cluster ) def from_busmap(x): @@ -134,7 +132,7 @@ def from_busmap(x): # Copy all lines whose `bus1` lies within the cluster right_external_connectors = filter_right_external_connector( - getattr(self.original_network, line_type), is_bus_in_cluster + rows, is_bus_in_cluster ) if not right_external_connectors.empty: ca_option = pd.get_option("mode.chained_assignment") From 7f61dbe562d5f41b471bc6d3d312bca9976c7651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 23 Mar 2023 18:38:49 +0100 Subject: [PATCH 08/49] Factor `timeseries` out into a local variable Same reasons as in the previous commit. --- etrago/cluster/disaggregation.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index b052ba45b..93d4be02f 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -94,6 +94,9 @@ def is_bus_in_cluster(conn, busflags=busflags): line_types = ["lines", "links", "transformers"] for line_type in line_types: rows: pd.DataFrame = getattr(self.original_network, line_type) + timeseries: dict[str, pd.DataFrame] = getattr( + self.original_network, line_type + "_t" + ) # Copy all lines that reside entirely inside the cluster ... setattr( partial_network, @@ -105,11 +108,7 @@ def is_bus_in_cluster(conn, busflags=busflags): # TODO: These are all time series, not just the ones from lines # residing entirely in side the cluster. # Is this a problem? - setattr( - partial_network, - line_type + "_t", - getattr(self.original_network, line_type + "_t"), - ) + setattr(partial_network, line_type + "_t", timeseries) # Copy all lines whose `bus0` lies within the cluster left_external_connectors = filter_left_external_connector( From ff9c359b27c807a89724ea676bf4976469814642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 22 Mar 2023 05:59:58 +0100 Subject: [PATCH 09/49] Use implicit string joining instead of `+` And move separating spaces to the beginning of the line while at it. --- etrago/cluster/disaggregation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 93d4be02f..50095e4f8 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -606,9 +606,9 @@ def solve_partial_network( pnb.loc[:, "p_nom_extendable"] == clb.iloc[0].at["p_nom_extendable"] ).all(), ( - "The `'p_nom_extendable'` flag for the current " - + "cluster's bus does not have the same value " - + "it has on the buses of it's partial network." + "The `'p_nom_extendable'` flag for the current" + " cluster's bus does not have the same value" + " it has on the buses of it's partial network." ) if clb.iloc[0].at["p_nom_extendable"]: From 4848efe77af4cdbbf8ba22416c9d390cd460728e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 00:12:49 +0100 Subject: [PATCH 10/49] Fix typo in comment: "in side" -> "inside" --- etrago/cluster/disaggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 50095e4f8..23a44d67e 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -106,7 +106,7 @@ def is_bus_in_cluster(conn, busflags=busflags): # ... and their time series # TODO: These are all time series, not just the ones from lines - # residing entirely in side the cluster. + # residing entirely inside the cluster. # Is this a problem? setattr(partial_network, line_type + "_t", timeseries) From aaa04359088142c5cdc8b19ec0e882871da98e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 00:43:07 +0100 Subject: [PATCH 11/49] Update comment about unfiltered time series --- etrago/cluster/disaggregation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 23a44d67e..b98428e34 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -108,6 +108,10 @@ def is_bus_in_cluster(conn, busflags=busflags): # TODO: These are all time series, not just the ones from lines # residing entirely inside the cluster. # Is this a problem? + # I hope not, because neither is `rows.index` a subset + # of the columns of one of the values of `timeseries`, + # nor the other way around, so it's not clear how to + # align both. setattr(partial_network, line_type + "_t", timeseries) # Copy all lines whose `bus0` lies within the cluster From 096f9d3c471807767d669374f6fb4d03e9f5b20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 00:53:58 +0100 Subject: [PATCH 12/49] Replace `DataFrame.append` with `pandas.concat` The `DataFrame.append` method is deprecated in favour of `pandas.concat` as of `pandas` version 1.4.0. --- etrago/cluster/disaggregation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index b98428e34..1904d6414 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -181,8 +181,8 @@ def from_busmap(x): self.reindex_with_prefix(externals_to_insert) # .. and insert them as well as their time series - partial_network.buses = partial_network.buses.append( - externals_to_insert + partial_network.buses = pd.concat( + [partial_network.buses, externals_to_insert] ) partial_network.buses_t = self.original_network.buses_t @@ -214,7 +214,9 @@ def from_busmap(x): setattr( partial_network, bustype, - getattr(partial_network, bustype).append(buses_to_insert), + pd.concat( + [getattr(partial_network, bustype), buses_to_insert] + ), ) # Also copy their time series From 6ae2a45fb8856348b7a2629afca7329bf430cbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 01:15:14 +0100 Subject: [PATCH 13/49] Improve sanity check's comprehensibility Use descriptive variable names for intermediate values to make the code shorter and basically read as what it does: assert that the `DataFrame` of rows which don't pass the sanity check, i.e. are "not sane", or `~sane`, is empty. --- etrago/cluster/disaggregation.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 1904d6414..b976b123e 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -246,16 +246,13 @@ def from_busmap(x): # Just a simple sanity check # TODO: Remove when sure that disaggregation will not go insane anymore for line_type in line_types: - assert ( - getattr(partial_network, line_type) - .bus0.isin(partial_network.buses.index) - .all() - ) - assert ( - getattr(partial_network, line_type) - .bus1.isin(partial_network.buses.index) - .all() - ) + rows = getattr(partial_network, line_type) + + sane = rows.bus0.isin(partial_network.buses.index) + assert rows.loc[~sane, :].empty + + sane = rows.bus1.isin(partial_network.buses.index) + assert rows.loc[~sane, :].empty return partial_network, external_buses From 4339ebedd22cb038518af0ed3009073982cb9c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 01:23:51 +0100 Subject: [PATCH 14/49] Add an error message to sanity checks --- etrago/cluster/disaggregation.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index b976b123e..4fe0b798e 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -249,10 +249,18 @@ def from_busmap(x): rows = getattr(partial_network, line_type) sane = rows.bus0.isin(partial_network.buses.index) - assert rows.loc[~sane, :].empty + assert rows.loc[~sane, :].empty, ( + f"Not all `partial_network.{line_type}.bus0` entries are" + f" contained in `partial_network.buses.index`." + f" Spurious additional rows:\nf{rows.loc[~sane, :]}" + ) sane = rows.bus1.isin(partial_network.buses.index) - assert rows.loc[~sane, :].empty + assert rows.loc[~sane, :].empty, ( + f"Not all `partial_network.{line_type}.bus1` entries are" + f" contained in `partial_network.buses.index`." + f" Spurious additional rows:\nf{rows.loc[~sane, :]}" + ) return partial_network, external_buses From 021d83bb99b73c1dc71655f9d5233ae68943e0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 01:26:33 +0100 Subject: [PATCH 15/49] Bulk insert disaggregated columns Instead of using a for loop to append them one at a time. Doing so triggered a few warnings about the `DataFrame` being fragmented. The new code should prevent that, while still inserting the same values. Additionally, no duplicate columns are created. Instead the data in already existing columns is overwritten, which is a plus. --- etrago/cluster/disaggregation.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 4fe0b798e..4d6deb4fa 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -680,9 +680,13 @@ def solve_partial_network( else () ) ws = weight.sum(axis=len(loc)) - for bus_id in filtered.index: - values = clt * weight.loc[loc + (bus_id,)] / ws - pn_t[s].insert(len(pn_t[s].columns), bus_id, values) + new_columns = pd.DataFrame( + { + bus_id: clt * weight.loc[loc + (bus_id,)] / ws + for bus_id in filtered.index + } + ) + pn_t[s].loc[:, new_columns.columns] = new_columns def transfer_results(self, *args, **kwargs): kwargs["bustypes"] = ["generators", "storage_units"] From f00d49aa9c6dd0a89364cc30fd541c2ef257d387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 01:45:17 +0100 Subject: [PATCH 16/49] Add some information to "etrago/tools/__init__.py" A short docstring along with author, copyright and license information. --- etrago/tools/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/etrago/tools/__init__.py b/etrago/tools/__init__.py index 497ec4c5a..df40335b9 100644 --- a/etrago/tools/__init__.py +++ b/etrago/tools/__init__.py @@ -1,8 +1,12 @@ -""" +"""Multi purpose tools that don't fit anywhere else in eTraGo. """ -__copyright__ = "tba" -__license__ = "tba" -__author__ = "tba" +__copyright__ = ( + "Copyright (C) 2023" + " Otto-von-Guericke-University Magdeburg," + " Research group for theoretical computer science" +) +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__author__ = "gnn " From 4ecd568dfc0de05d1b8a9832f1c63a9864708c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 03:52:00 +0100 Subject: [PATCH 17/49] Add various ways of doing nothing --- etrago/tools/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/etrago/tools/__init__.py b/etrago/tools/__init__.py index df40335b9..170b87144 100644 --- a/etrago/tools/__init__.py +++ b/etrago/tools/__init__.py @@ -10,3 +10,29 @@ __author__ = "gnn " +def noop(*ignored_arguments, **ignored_keyword_arguments): + """Do nothing. + + Accept all kinds of arguments, ignore them and do nothing. + """ + pass + + +class Noops: + """Provide arbitrarily named methods that do nothing. + + Any attribute access will return a method that does nothing, i.e. + all methods of this object are :py:func:`noop`s. Normally you don't + need to instantiate this class. All instances behave the same, so + the containing module provides one called :py:obj:`noops` which you + can import and use. + """ + + @classmethod + def __getattribute__(cls, ignored_name): + return noop + + +noops = Noops() +"""A default :py:class:`Noops` instance so you don't have to create one. +""" From 17c5e631b6f11d3f05efdc0eb69cd43c6fbfac1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 03:56:41 +0100 Subject: [PATCH 18/49] Disable profiling more thorough Using `noops` not only disables the profiling report, but also makes `profile.enable()` and `profile.disable()` calls do nothing with as little overhead as possible. --- etrago/cluster/disaggregation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 4d6deb4fa..cd0ef2995 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -9,6 +9,7 @@ from pypsa import Network import pandas as pd +from etrago.tools import noops from etrago.tools.utilities import residual_load @@ -283,6 +284,7 @@ def solve(self, scenario, solver): ) } profile = cProfile.Profile() + profile = noops for i, cluster in enumerate(sorted(clusters)): log.info("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n)) profile.enable() @@ -354,7 +356,7 @@ def solve(self, scenario, solver): self.stats["check"] = time.time() - t log.info("Checks computed in {self.stats['check']}") - # profile.print_stats(sort='cumtime') + profile.print_stats(sort="cumtime") def transfer_results( self, From c350ae4f7b26e9c09ddd243cff4fe713cb399092 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 12 Jan 2023 09:53:08 +0100 Subject: [PATCH 19/49] Don't create `disaggregated_network.results` Results are stored directly in `disaggregated_network`'s attributes. --- etrago/cluster/disaggregation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index cd0ef2995..ecfa7f8dd 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -766,7 +766,6 @@ def run_disaggregation(self): self.disaggregated_network.generators_t.p.fillna(0, inplace=True) self.disaggregated_network.generators_t.q.fillna(0, inplace=True) - self.disaggregated_network.results = self.network.results log.info( "Time for overall desaggregation [min]: {:.2}".format( (time.time() - t) / 60 From 9db0c2d67163ad99a65119f74b0d89c5a4806c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 3 Mar 2023 16:51:19 +0100 Subject: [PATCH 20/49] Use a shorter way of getting the first index value That's what `next(clb.itertuples()).Index` does. It just gets the index of the first row of `clb`, so `clb.index[0]` should be equivalent but way clearer and more readable. Also note that `clb` should only have one row anyway. --- etrago/cluster/disaggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index ecfa7f8dd..2f45e9d0f 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -665,7 +665,7 @@ def solve_partial_network( if s in self.skip: continue filtered = pnb.loc[filters.get(s, slice(None))] - clt = cl_t[s].loc[:, next(clb.itertuples()).Index] + clt = cl_t[s].loc[:, clb.index[0]] weight = reduce( multiply, ( From 57352b106e6b193f98be44c8935003b2ca00f7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 22 Mar 2023 06:04:34 +0100 Subject: [PATCH 21/49] Replace older string interpolation with f-strings --- etrago/cluster/disaggregation.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 2f45e9d0f..79e614140 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -286,7 +286,7 @@ def solve(self, scenario, solver): profile = cProfile.Profile() profile = noops for i, cluster in enumerate(sorted(clusters)): - log.info("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n)) + log.info(f"Decompose {cluster=} ({i + 1}/{n})") profile.enable() t = time.time() partial_network, externals = self.construct_partial_network( @@ -326,9 +326,7 @@ def solve(self, scenario, solver): ("generators", {"p": fs, "q": fs}), ("storage_units", {"p": fs, "state_of_charge": fs, "q": fs}), ): - log.info( - "Attribute sums, {}, clustered - disaggregated:".format(bt) - ) + log.info(f"Attribute sums, {bt}, clustered - disaggregated:") cnb = getattr(self.clustered_network, bt) onb = getattr(self.original_network, bt) log.info( @@ -340,7 +338,7 @@ def solve(self, scenario, solver): ) ) - log.info("Series sums, {}, clustered - disaggregated:".format(bt)) + log.info(f"Series sums, {bt}, clustered - disaggregated:") cnb = getattr(self.clustered_network, bt + "_t") onb = getattr(self.original_network, bt + "_t") for s in ts: @@ -575,10 +573,8 @@ def solve_partial_network( if len(clb) == 0: continue assert len(clb) == 1, ( - "Cluster {} has {} buses for group {}.\n".format( - cluster, len(clb), group - ) - + "Should be exactly one." + f"Cluster {cluster} has {len(clb)} buses for {group=}." + "\nShould be exactly one." ) # Remove buses not belonging to the partial network pnb = pn_buses.iloc[ From 988d19cb95e8811d4e6e425f89952d58eb992468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 24 Mar 2023 05:33:50 +0100 Subject: [PATCH 22/49] Disaggregate "p0" and "p1" of "links" Instead of only retaining connecting components, i.e. "lines", "links" and "transformers", which are completely contained in a cluster, retain all components which have at least one endpoint inside the cluster. This is what the switch from `&` to `|` does. That way, these components are available as `cl_buses` when running `solve_partial_network`. Then, when solving the partial network, only select those components for disaggregation which have the left endpoint, i.e. `bus0` in the cluster and group them by right endpoint, i.e. `bus1`. Adjust sanity checks accordingly, as now a component doesn't have to have both of its endpoints inside the cluster, but only needs to have one. --- etrago/cluster/disaggregation.py | 36 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 79e614140..6d5a89b5c 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -249,18 +249,13 @@ def from_busmap(x): for line_type in line_types: rows = getattr(partial_network, line_type) - sane = rows.bus0.isin(partial_network.buses.index) - assert rows.loc[~sane, :].empty, ( - f"Not all `partial_network.{line_type}.bus0` entries are" + left = rows.bus0.isin(partial_network.buses.index) + right = rows.bus1.isin(partial_network.buses.index) + assert rows.loc[~(left | right), :].empty, ( + f"Not all `partial_network.{line_type}` have an endpoint," + " i.e. `bus0` or `bus1`," f" contained in `partial_network.buses.index`." - f" Spurious additional rows:\nf{rows.loc[~sane, :]}" - ) - - sane = rows.bus1.isin(partial_network.buses.index) - assert rows.loc[~sane, :].empty, ( - f"Not all `partial_network.{line_type}.bus1` entries are" - f" contained in `partial_network.buses.index`." - f" Spurious additional rows:\nf{rows.loc[~sane, :]}" + f" Spurious additional rows:\nf{rows.loc[~(left | right), :]}" ) return partial_network, external_buses @@ -530,6 +525,10 @@ def solve_partial_network( ): log.debug("Solving partial network.") bustypes = { + "links": { + "group_by": ("carrier", "bus1"), + "series": ("p0", "p1"), + }, "generators": {"group_by": ("carrier",), "series": ("p", "q")}, "storage_units": { "group_by": ("carrier", "max_hours"), @@ -546,6 +545,8 @@ def solve_partial_network( ) else ("p_nom_opt", "p_max_pu") ), + "p0": ("p_nom_opt",), + "p1": ("p_nom_opt",), "state_of_charge": ("p_nom_opt",), } filters = {"q": lambda o: o.control == "PV"} @@ -565,7 +566,11 @@ def solve_partial_network( ] ) for group in groups: - clb = cl_buses[cl_buses.bus == cluster] + clb = ( + cl_buses[cl_buses.bus == cluster] + if "bus" in cl_buses.columns + else cl_buses[cl_buses.bus0 == cluster] + ) query = " & ".join( ["({key} == {value!r})".format(**axis) for axis in group] ) @@ -581,7 +586,10 @@ def solve_partial_network( [ i for i, row in enumerate(pn_buses.itertuples()) - if not row.bus.startswith(self.idx_prefix) + for bus in [ + row.bus if hasattr(row, "bus") else row.bus1 + ] + if not bus.startswith(self.idx_prefix) ] ] pnb = pnb.query(query) @@ -701,7 +709,7 @@ def swap_series(s): def filter_internal_connector(conn, is_bus_in_cluster): return conn[ - conn.bus0.apply(is_bus_in_cluster) & conn.bus1.apply(is_bus_in_cluster) + conn.bus0.apply(is_bus_in_cluster) | conn.bus1.apply(is_bus_in_cluster) ] From b6f42cd6b4f7f2e6fd343fb7d9537a9d908bcba2 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 11 Jan 2023 18:33:23 +0100 Subject: [PATCH 23/49] Disaggregate "stores" Differentiate between the different columns of stores and other components, as stores use "e_.*" where other components use "p_.*". --- etrago/cluster/disaggregation.py | 49 +++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 6d5a89b5c..fe856b3aa 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -534,6 +534,10 @@ def solve_partial_network( "group_by": ("carrier", "max_hours"), "series": ("p", "state_of_charge", "q"), }, + "stores": { + "group_by": ("carrier",), + "series": ("e", "p"), + }, } weights = { "p": ("p_nom_opt", "p_max_pu"), @@ -548,14 +552,31 @@ def solve_partial_network( "p0": ("p_nom_opt",), "p1": ("p_nom_opt",), "state_of_charge": ("p_nom_opt",), + "e": ("e_nom_opt",), } filters = {"q": lambda o: o.control == "PV"} + for bustype in bustypes: + # Define attributeof components which are available + if bustype == "stores": + extendable_flag = "e_nom_extendable" + nominal_capacity = "e_nom" + optimal_capacity = "e_nom_opt" + maximal_capacity = "e_nom_max" + weights["p"] = ("e_nom_opt", "e_max_pu") + else: + extendable_flag = "p_nom_extendable" + nominal_capacity = "p_nom" + optimal_capacity = "p_nom_opt" + maximal_capacity = "p_nom_max" + weights["p"] = ("p_nom_opt", "p_max_pu") + log.debug(f"Decomposing {bustype}.") pn_t = getattr(partial_network, bustype + "_t") cl_t = getattr(self.clustered_network, bustype + "_t") pn_buses = getattr(partial_network, bustype) cl_buses = getattr(self.clustered_network, bustype) + groups = product( *[ [ @@ -603,8 +624,8 @@ def solve_partial_network( ) if not ( - pnb.loc[:, "p_nom_extendable"].all() - or not pnb.loc[:, "p_nom_extendable"].any() + pnb.loc[:, extendable_flag].all() + or not pnb.loc[:, extendable_flag].any() ): raise NotImplementedError( "The `'p_nom_extendable'` flag for buses in the" @@ -620,41 +641,41 @@ def solve_partial_network( ) else: assert ( - pnb.loc[:, "p_nom_extendable"] - == clb.iloc[0].at["p_nom_extendable"] + pnb.loc[:, extendable_flag] + == clb.iloc[0].at[extendable_flag] ).all(), ( "The `'p_nom_extendable'` flag for the current" " cluster's bus does not have the same value" " it has on the buses of it's partial network." ) - if clb.iloc[0].at["p_nom_extendable"]: + if clb.iloc[0].at[extendable_flag]: # That means, `p_nom` got computed via optimization and we # have to distribute it into the subnetwork first. - pnb_p_nom_max = pnb.loc[:, "p_nom_max"] + pnb_p_nom_max = pnb.loc[:, maximal_capacity] p_nom_max_global = pnb_p_nom_max.sum(axis="index") - pnb.loc[:, "p_nom_opt"] = ( - clb.iloc[0].at["p_nom_opt"] + pnb.loc[:, optimal_capacity] = ( + clb.iloc[0].at[optimal_capacity] * pnb_p_nom_max / p_nom_max_global ) getattr(self.original_network, bustype).loc[ - pnb.index, "p_nom_opt" - ] = pnb.loc[:, "p_nom_opt"] - pnb.loc[:, "p_nom"] = pnb.loc[:, "p_nom_opt"] + pnb.index, optimal_capacity + ] = pnb.loc[:, optimal_capacity] + pnb.loc[:, nominal_capacity] = pnb.loc[:, optimal_capacity] else: # That means 'p_nom_opt' didn't get computed and is # potentially not present in the dataframe. But we want to # always use 'p_nom_opt' in the remaining code, so save a # view of the computed 'p_nom' values under 'p_nom_opt'. - pnb.loc[:, "p_nom_opt"] = pnb.loc[:, "p_nom"] + pnb.loc[:, optimal_capacity] = pnb.loc[:, nominal_capacity] # This probably shouldn't be here, but rather in # `transfer_results`, but it's easier to do it this way right # now. getattr(self.original_network, bustype).loc[ - pnb.index, "p_nom_opt" - ] = pnb.loc[:, "p_nom_opt"] + pnb.index, optimal_capacity + ] = pnb.loc[:, optimal_capacity] timed = ( lambda key, series=set( s From f109cec35b0e4713564ab0ebcf960e98b79b4bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Mon, 16 Jan 2023 17:20:48 +0100 Subject: [PATCH 24/49] Transfer `"links"`' and `"stores"`' results --- etrago/cluster/disaggregation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index fe856b3aa..f41453b3f 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -716,10 +716,12 @@ def solve_partial_network( pn_t[s].loc[:, new_columns.columns] = new_columns def transfer_results(self, *args, **kwargs): - kwargs["bustypes"] = ["generators", "storage_units"] + kwargs["bustypes"] = ["generators", "links", "storage_units", "stores"] kwargs["series"] = { "generators": {"p"}, + "links": {"p0", "p1"}, "storage_units": {"p", "state_of_charge"}, + "stores": {"e", "p"}, } return super().transfer_results(*args, **kwargs) From 886211bf49cc34b96108ec63bc0c40b455b66adc Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 6 Feb 2023 14:52:14 +0100 Subject: [PATCH 25/49] Replace infinite values with a large constant Use a large constant to avoid NaNs as disaggregation weights. --- etrago/cluster/disaggregation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index f41453b3f..78f5cc755 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -653,7 +653,13 @@ def solve_partial_network( # That means, `p_nom` got computed via optimization and we # have to distribute it into the subnetwork first. pnb_p_nom_max = pnb.loc[:, maximal_capacity] + + # If upper limit is infinite, replace it by a very large + # number to avoid NaN values in the calculation + pnb_p_nom_max.replace(float("inf"), 10000000, inplace=True) + p_nom_max_global = pnb_p_nom_max.sum(axis="index") + pnb.loc[:, optimal_capacity] = ( clb.iloc[0].at[optimal_capacity] * pnb_p_nom_max From 854efd9814d7e028ee9d42984d40ca5b4b1006f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Wed, 8 Mar 2023 16:36:14 +0100 Subject: [PATCH 26/49] Reformat the definition of `timed` Just putting `key in series` in parenthesis prevents `black` from spreading it over multiple lines, which means that the whole `lambda` expression doesn't have to be put in parenthesis, making the whole thing less indented and a bit more readable IMHO. Also note that it uses `{}` instead of `set()` now. Last but not least, use a `# noqa` comment do pacify linters complaining about an assigned lambda. A `def` would make these lines less compact and thus slightly less readable, IMHO. --- etrago/cluster/disaggregation.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 78f5cc755..2c18601c7 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -682,15 +682,12 @@ def solve_partial_network( getattr(self.original_network, bustype).loc[ pnb.index, optimal_capacity ] = pnb.loc[:, optimal_capacity] - timed = ( - lambda key, series=set( - s - for s in cl_t - if not cl_t[s].empty - if not pn_t[s].columns.intersection(pnb.index).empty - ): key - in series - ) + timed = lambda key, series={ # noqa: 731 + s + for s in cl_t + if not cl_t[s].empty + if not pn_t[s].columns.intersection(pnb.index).empty + }: (key in series) for s in bustypes[bustype]["series"]: if s in self.skip: From 73b9e3b356ef1a6df923204d424cc7a9792e6e46 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 27 Mar 2023 09:39:52 +0200 Subject: [PATCH 27/49] Add busmap as type pandas.Series --- etrago/cluster/disaggregation.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 2c18601c7..c727fc479 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -26,10 +26,11 @@ def __init__( self.original_network = original_network self.clustered_network = clustered_network self.clustering = clustering + self.busmap = pd.Series(clustering.busmap) self.buses = pd.merge( original_network.buses, - clustering.busmap.to_frame(name="cluster"), + self.busmap.to_frame(name="cluster"), left_index=True, right_index=True, ) @@ -121,7 +122,7 @@ def is_bus_in_cluster(conn, busflags=busflags): ) def from_busmap(x): - return self.idx_prefix + self.clustering.busmap.loc[x] + return self.idx_prefix + self.busmap.loc[x] if not left_external_connectors.empty: ca_option = pd.get_option("mode.chained_assignment") @@ -270,7 +271,7 @@ def solve(self, scenario, solver): :param scenario: :param solver: Solver that may be used to optimize partial networks """ - clusters = set(self.clustering.busmap.values) + clusters = set(self.busmap.values) n = len(clusters) self.stats = { "clusters": pd.DataFrame( @@ -424,7 +425,7 @@ def _validate_disaggregation_generators(self, cluster, f): def extra_functionality(network, snapshots): f(network, snapshots) generators = self.original_network.generators.assign( - bus=lambda df: df.bus.map(self.clustering.busmap) + bus=lambda df: df.bus.map(self.busmap) ) def construct_constraint(model, snapshot, carrier): @@ -476,7 +477,7 @@ def extra_functionality(network, snapshots): ]: generators = getattr( self.original_network, bustype_pypsa - ).assign(bus=lambda df: df.bus.map(self.clustering.busmap)) + ).assign(bus=lambda df: df.bus.map(self.busmap)) for suffix in suffixes: def construct_constraint(model, snapshot): From d806f8b556cd082d18097aaa0d51a213e2561d4f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 27 Mar 2023 09:40:29 +0200 Subject: [PATCH 28/49] Un-comment spatial disaggregation --- etrago/appl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index da10a4e3d..ee851d3a6 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -134,7 +134,7 @@ }, }, "network_clustering_ehv": False, # clustering of HV buses to EHV buses. - "disaggregation": None, # None, 'mini' or 'uniform' + "disaggregation": "uniform", # None, 'mini' or 'uniform' # Temporal Complexity: "snapshot_clustering": { "active": False, # choose if clustering is activated @@ -517,7 +517,7 @@ def run_etrago(args, json_path): # spatial disaggregation # needs to be adjusted for new sectors - # etrago.disaggregation() + etrago.disaggregation() # calculate central etrago results etrago.calc_results() From 95c22f016fba3dfb22074fa7b5c0c7832a089482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 20 Apr 2023 17:09:42 +0200 Subject: [PATCH 29/49] Replace `.busmap` with `.buses.loc[:, "cluster"]` This partially reverts commit 73b9e3b3. The `.busmap` attribute isn't really necessary, as `.buses.loc[:, "cluster"]` returns the same `Series`, i.e. the same indices, values and mapping of index to value. The only difference is in the order of the entries, which shouldn't matter. --- etrago/cluster/disaggregation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index c727fc479..296977aa2 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -26,11 +26,10 @@ def __init__( self.original_network = original_network self.clustered_network = clustered_network self.clustering = clustering - self.busmap = pd.Series(clustering.busmap) self.buses = pd.merge( original_network.buses, - self.busmap.to_frame(name="cluster"), + self.clustering.busmap.to_frame(name="cluster"), left_index=True, right_index=True, ) @@ -122,7 +121,7 @@ def is_bus_in_cluster(conn, busflags=busflags): ) def from_busmap(x): - return self.idx_prefix + self.busmap.loc[x] + return self.idx_prefix + self.buses.loc[x, "cluster"] if not left_external_connectors.empty: ca_option = pd.get_option("mode.chained_assignment") @@ -271,7 +270,7 @@ def solve(self, scenario, solver): :param scenario: :param solver: Solver that may be used to optimize partial networks """ - clusters = set(self.busmap.values) + clusters = set(self.buses.loc[:, "cluster"].values) n = len(clusters) self.stats = { "clusters": pd.DataFrame( @@ -425,7 +424,7 @@ def _validate_disaggregation_generators(self, cluster, f): def extra_functionality(network, snapshots): f(network, snapshots) generators = self.original_network.generators.assign( - bus=lambda df: df.bus.map(self.busmap) + bus=lambda df: df.bus.map(self.buses.loc[:, "cluster"]) ) def construct_constraint(model, snapshot, carrier): @@ -477,7 +476,9 @@ def extra_functionality(network, snapshots): ]: generators = getattr( self.original_network, bustype_pypsa - ).assign(bus=lambda df: df.bus.map(self.busmap)) + ).assign( + bus=lambda df: df.bus.map(self.buses.loc[:, "cluster"]) + ) for suffix in suffixes: def construct_constraint(model, snapshot): From 0a9c5156358c267728798f2f4c8c5734a396993f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 18 Apr 2023 17:47:26 +0200 Subject: [PATCH 30/49] Filter links by `bus0` instead of `bus1` This fixes a bug in the original `p0`, `p1` disaggregation implementation for `links` in commit 988d19cb9. For components having a `bus0` and a `bus1` instead of just a `bus`, `bus0` is taken as corresponding to `bus`. So filtering out these components should be done by looking at `bus0`, not `bus1`. --- etrago/cluster/disaggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 296977aa2..67b77522a 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -610,7 +610,7 @@ def solve_partial_network( i for i, row in enumerate(pn_buses.itertuples()) for bus in [ - row.bus if hasattr(row, "bus") else row.bus1 + row.bus if hasattr(row, "bus") else row.bus0 ] if not bus.startswith(self.idx_prefix) ] From 7a5a100eac7b1464a07aa555893f23dd67330ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 21 Apr 2023 06:18:14 +0200 Subject: [PATCH 31/49] Move `bus == cluster` filtering out of the loop Nothing in the filter uses the loop variable, so the filtering can be moved out of the loop. Also, since `cl_buses` is only used to determine the value of `clb` for the duration of the loop and nowhere else, filtering can be done directly when assigning `cl_buses` and `clb` can be determined directly by querying the filtered DataFrame. Filtering before determining the groups massively speeds up the disaggregation. --- etrago/cluster/disaggregation.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 67b77522a..06d044318 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -577,8 +577,10 @@ def solve_partial_network( pn_t = getattr(partial_network, bustype + "_t") cl_t = getattr(self.clustered_network, bustype + "_t") pn_buses = getattr(partial_network, bustype) - cl_buses = getattr(self.clustered_network, bustype) - + cl_buses = getattr(self.clustered_network, bustype)[ + lambda df: df.loc[:, "bus" if "bus" in df.columns else "bus0"] + == cluster + ] groups = product( *[ [ @@ -589,15 +591,10 @@ def solve_partial_network( ] ) for group in groups: - clb = ( - cl_buses[cl_buses.bus == cluster] - if "bus" in cl_buses.columns - else cl_buses[cl_buses.bus0 == cluster] - ) query = " & ".join( ["({key} == {value!r})".format(**axis) for axis in group] ) - clb = clb.query(query) + clb = cl_buses.query(query) if len(clb) == 0: continue assert len(clb) == 1, ( From a17f05dface0fc9eeb27f99c98179818b1b9bbac Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 18 Apr 2023 17:45:17 +0200 Subject: [PATCH 32/49] Adjust `query` for the special case of `"links"` For links, the `query` contains references to `bus1`, which has a different meaning depending on whether we are dealing with a cluster or the partial network the cluster represents. For clusters, the query is correct, because the group was generated by looking up the `bus1` values of links attached to the cluster. But to find the corresponding links in the partial network, one has to look for those links whose `bus1` value matches the index of one of the buses whose `"cluster"` entry is equal to the group's `bus1` value. Or in other words, since the group's `bus1` value identifies a cluster, one has to - first find all buses corresponding to that cluster, - then find all links whose `bus1` value points to one of these buses. --- etrago/cluster/disaggregation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 06d044318..45e089093 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -612,6 +612,14 @@ def solve_partial_network( if not bus.startswith(self.idx_prefix) ] ] + if bustype == "links": + index = self.buses[ + self.buses.loc[:, "cluster"] == group[1]["value"] + ].index.tolist() + query = ( + f"(carrier == {group[0]['value']!r})" + f" & (bus1 in {index})" + ) pnb = pnb.query(query) assert not pnb.empty, ( "Cluster has a bus for:" From e579d20dec8004fea643328a9fd7417aa17b5659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 21 Apr 2023 06:49:59 +0200 Subject: [PATCH 33/49] Use values in cluster columns to define groups This is the correct way anyway and fixes a bug, since the values stored in each group (should) uniquely identify a cluster. The bug just never surfaced, as these values are usually also found in the partial network. This isn't the case for the values of `bus`, `bus0` and `bus1` though, because these point to other cluster or partial network buses, so they differ depending on whether the attribute is on a cluster or a partial network component, so the bug was finally caught. --- etrago/cluster/disaggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 45e089093..abb6f3665 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -585,7 +585,7 @@ def solve_partial_network( *[ [ {"key": key, "value": value} - for value in set(pn_buses.loc[:, key]) + for value in set(cl_buses.loc[:, key]) ] for key in bustypes[bustype]["group_by"] ] From 12a1933d996e33a713fd12d7d4d0a218eeadfb7f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 18 Apr 2023 17:46:48 +0200 Subject: [PATCH 34/49] Allow `pnb` to be empty in certain cases --- etrago/cluster/disaggregation.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index abb6f3665..a4144ae86 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -621,7 +621,23 @@ def solve_partial_network( f" & (bus1 in {index})" ) pnb = pnb.query(query) - assert not pnb.empty, ( + assert not pnb.empty or ( + # In some cases, a district heating grid is connected to a + # substation only via a resistive_heater but not e.g. by a + # heat_pump or one of the other listed `carrier`s. + # In the clustered network, there are both. + # In these cases, the `pnb` can actually be empty. + group[0]["value"] + in [ + "central_gas_boiler", + "central_heat_pump", + "central_gas_CHP_heat", + "central_gas_CHP", + "CH4", + "DC", + "OCGT", + ] + ), ( "Cluster has a bus for:" + "\n ".join( ["{key}: {value!r}".format(**axis) for axis in group] @@ -629,6 +645,8 @@ def solve_partial_network( + "\nbut no matching buses in its corresponding " + "partial network." ) + if pnb.empty: + continue if not ( pnb.loc[:, extendable_flag].all() From 774ee1a7f4ec192fd7eeb55a9ea071b8b0bef956 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 19 Apr 2023 09:30:02 +0200 Subject: [PATCH 35/49] Assert that time series sum matches cluster value --- etrago/cluster/disaggregation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index a4144ae86..407e03169 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -741,6 +741,12 @@ def solve_partial_network( for bus_id in filtered.index } ) + delta = abs((new_columns.sum(axis=1) - clt).sum()) + epsilon = 1e-5 + assert delta < epsilon, ( + "Sum of disaggregated time series does not match" + f" aggregated timeseries: {delta=} > {epsilon=}." + ) pn_t[s].loc[:, new_columns.columns] = new_columns def transfer_results(self, *args, **kwargs): From b45c1d36e51f66879429b4ed06e18b0913f4cabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 21 Apr 2023 11:40:40 +0200 Subject: [PATCH 36/49] Make string containing "{expression}" an f-string It was obviously meant that way, so this is just a bugfix. Add punctuation and a unit for better readability, too. --- etrago/cluster/disaggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index 407e03169..dfd635ac6 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -347,7 +347,7 @@ def solve(self, scenario, solver): ) profile.disable() self.stats["check"] = time.time() - t - log.info("Checks computed in {self.stats['check']}") + log.info(f"Checks computed in {self.stats['check']}s.") profile.print_stats(sort="cumtime") From 71e2f8c81f03758f4ce1f6fd4d6f4f0af9d84c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Sun, 23 Apr 2023 22:54:35 +0200 Subject: [PATCH 37/49] Add links to "clustered - disaggregated" checks This code prints the differences of the sum of all values of a certain attribute or timeseries in the clustered network to the sum of the corresponding disaggregated values in the original network. Usually these should be close to zero. For `links`'s "p0" and "p1" timeseries these aren't close to zero, but that's OK as these include values from the CH4 links, which aren't disaggregated. But it's probably still a good idea to have those printed in order to be able to check whether these differences are in the correct ballpark. Maybe the check can later be extended to be filtered so that they are close to zero again. --- etrago/cluster/disaggregation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py index dfd635ac6..054525fb5 100644 --- a/etrago/cluster/disaggregation.py +++ b/etrago/cluster/disaggregation.py @@ -320,6 +320,7 @@ def solve(self, scenario, solver): for bt, ts in ( ("generators", {"p": fs, "q": fs}), ("storage_units", {"p": fs, "state_of_charge": fs, "q": fs}), + ("links", {"p0": fs, "p1": fs}), ): log.info(f"Attribute sums, {bt}, clustered - disaggregated:") cnb = getattr(self.clustered_network, bt) From 3fed0145aa8ac7e37e7f9050a7b8ec9b955d9519 Mon Sep 17 00:00:00 2001 From: Katharina Esterl Date: Mon, 8 May 2023 10:16:06 +0200 Subject: [PATCH 38/49] Fix export of information when using segmentation --- etrago/cluster/snapshot.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/etrago/cluster/snapshot.py b/etrago/cluster/snapshot.py index daa0b6166..a5c48936e 100644 --- a/etrago/cluster/snapshot.py +++ b/etrago/cluster/snapshot.py @@ -653,7 +653,10 @@ def run( ) if segmentation != False: - timeseries.to_csv( + pd.DataFrame( + timeseries.reset_index(), + columns=["dates", "SegmentNo", "SegmentDuration"], + ).set_index("SegmentNo").to_csv( "timeseries_segmentation=" + str(segment_no) + ".csv" ) else: @@ -698,7 +701,7 @@ def prepare_pypsa_timeseries(network): Timeseries to be considered when clustering. """ - + loads = network.loads_t.p_set.copy() loads.columns = "L" + loads.columns @@ -795,7 +798,7 @@ def skip_snapshots(self): None. """ - + # save second network for optional dispatch disaggregation if ( self.args["temporal_disaggregation"]["active"] == True @@ -806,7 +809,6 @@ def skip_snapshots(self): n_skip = self.args["skip_snapshots"] if n_skip: - last_weight = ( int( ( @@ -834,4 +836,3 @@ def skip_snapshots(self): self.network.snapshot_weightings.loc[ self.network.snapshot_weightings.index[-1] ]["generators"] = last_weight - From 012d51cfea8c377617b927329e1b353bff96da98 Mon Sep 17 00:00:00 2001 From: ulfmueller Date: Wed, 17 May 2023 11:28:16 +0200 Subject: [PATCH 39/49] changed end_snapshot --- etrago/appl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etrago/appl.py b/etrago/appl.py index 0e6724009..76b84f4bb 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -63,7 +63,7 @@ "q_allocation": "p_nom", # allocate reactive power via 'p_nom' or 'p' }, "start_snapshot": 1, - "end_snapshot": 2, + "end_snapshot": 10, "solver": "gurobi", # glpk, cplex or gurobi "solver_options": { "BarConvTol": 1.0e-5, From 3c3b19634db909aa2d384942fc951a003e7f6d15 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 28 Jun 2023 14:38:24 +0200 Subject: [PATCH 40/49] include status2019 as a valid scn name --- etrago/tools/utilities.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 575f07761..66b37ecd6 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -2336,7 +2336,9 @@ def check_args(etrago): """ - names = ["eGon2035", "eGon100RE", "eGon2035_lowflex", "eGon100RE_lowflex"] + names = ["eGon2035", "eGon100RE", "eGon2035_lowflex", "eGon100RE_lowflex", + "status2019"] + assert ( etrago.args["scn_name"] in names ), f"'scn_name' has to be in {names} but is {etrago.args['scn_name']}." From c9193f559fdabefcdf6942065171b709de627eb6 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 28 Jun 2023 14:40:30 +0200 Subject: [PATCH 41/49] make H2 buses optional --- etrago/tools/network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etrago/tools/network.py b/etrago/tools/network.py index 97ffbfe77..a5900238f 100644 --- a/etrago/tools/network.py +++ b/etrago/tools/network.py @@ -324,7 +324,8 @@ def build_network_from_db(self): self.decommissioning() - self.add_ch4_h2_correspondence() + if "H2" in self.network.buses.carrier: + self.add_ch4_h2_correspondence() logger.info("Imported network from db") From c8d7427aa12f375212d9972c48be5e07b899294b Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 10 Jul 2023 13:47:20 +0200 Subject: [PATCH 42/49] REVERT-just for testing purposes --- etrago/appl.py | 54 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index ab64b5e17..6a6158f58 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -50,7 +50,7 @@ args = { # Setup and Configuration: - "db": "egon-data", # database session + "db": "powerd-data-server", # database session "gridversion": None, # None for model_draft or Version number "method": { # Choose method and settings for optimization "type": "lopf", # type of optimization, currently only 'lopf' @@ -63,7 +63,7 @@ "q_allocation": "p_nom", # allocate reactive power via 'p_nom' or 'p' }, "start_snapshot": 1, - "end_snapshot": 10, + "end_snapshot": 8760, "solver": "gurobi", # glpk, cplex or gurobi "solver_options": { "BarConvTol": 1.0e-5, @@ -74,7 +74,7 @@ "threads": 4, }, "model_formulation": "kirchhoff", # angles or kirchhoff - "scn_name": "eGon2035", # scenario: eGon2035 or eGon100RE + "scn_name": "status2019", # scenario: eGon2035 or eGon100RE # Scenario variations: "scn_extension": None, # None or array of extension scenarios "scn_decommissioning": None, # None or decommissioning scenario @@ -155,7 +155,7 @@ }, # Simplifications: "branch_capacity_factor": {"HV": 0.5, "eHV": 0.7}, # p.u. branch derating - "load_shedding": True, # meet the demand at value of loss load cost + "load_shedding": False, # meet the demand at value of loss load cost "foreign_lines": { "carrier": "AC", # 'DC' for modeling foreign lines as links "capacity": "osmTGmod", # 'osmTGmod', 'tyndp2020', 'ntc_acer' or 'thermal_acer' @@ -455,10 +455,52 @@ def run_etrago(args, json_path): # import network from database etrago.build_network_from_db() - + # adjust network regarding eTraGo setting etrago.adjust_network() - + breakpoint() + + # Generators + gen = etrago.network.generators + gen["country"] = gen.bus.map(etrago.network.buses.country) + import pandas as pd + ins_cap = pd.DataFrame() + for (country, carrier), df in gen.groupby(["country", "carrier"]): + ins_cap.at[country, carrier] = df.p_nom.sum() + #ins_cap.drop(columns=["load shedding"], inplace=True) + + # Loads + load = etrago.network.loads + load_t = etrago.network.loads_t.p_set + load["country"] = load.bus.map(etrago.network.buses.country) + load_country = pd.DataFrame() + for (country, carrier), df in load.groupby(["country", "carrier"]): + loads_id = df.index + load_country.at[country, carrier] = load_t[loads_id].sum().sum() + + # storage_units + sto = etrago.network.storage_units + sto["country"] = sto.bus.map(etrago.network.buses.country) + sto_country = pd.DataFrame() + for (country, carrier), df in sto.groupby(["country", "carrier"]): + sto_country.at[country, carrier] = df.p_nom.sum() + + # links + link = etrago.network.links + link_t = etrago.network.links_t + link_with_t = link.loc[link.index.isin(link_t["p_max_pu"].columns)] + + # buses + bus = etrago.network.buses + bus_country = pd.DataFrame() + for (country, carrier), df in bus.groupby(["country", "carrier"]): + bus_country.at[country, carrier] = len(df) + + # subnetworks + etrago.network.determine_network_topology() + sub_network = etrago.network.buses.loc[etrago.network.buses.carrier=="AC"] + sub_network = sub_network.loc[sub_network.country=="DE"] + sub_network.sub_network.value_counts() # ehv network clustering etrago.ehv_clustering() From f4faddd7c4c83a275558cd25fda1f2a7b3ae7096 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 17 Jul 2023 08:41:07 +0200 Subject: [PATCH 43/49] more checks for status2019 --- etrago/appl.py | 80 +++++++++++++++++++++---------------------- etrago/cluster/gas.py | 2 +- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 6a6158f58..a539752fb 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -63,7 +63,7 @@ "q_allocation": "p_nom", # allocate reactive power via 'p_nom' or 'p' }, "start_snapshot": 1, - "end_snapshot": 8760, + "end_snapshot": 10, "solver": "gurobi", # glpk, cplex or gurobi "solver_options": { "BarConvTol": 1.0e-5, @@ -458,51 +458,51 @@ def run_etrago(args, json_path): # adjust network regarding eTraGo setting etrago.adjust_network() - breakpoint() + #breakpoint() - # Generators - gen = etrago.network.generators - gen["country"] = gen.bus.map(etrago.network.buses.country) - import pandas as pd - ins_cap = pd.DataFrame() - for (country, carrier), df in gen.groupby(["country", "carrier"]): - ins_cap.at[country, carrier] = df.p_nom.sum() - #ins_cap.drop(columns=["load shedding"], inplace=True) + # # Generators + # gen = etrago.network.generators + # gen["country"] = gen.bus.map(etrago.network.buses.country) + # import pandas as pd + # ins_cap = pd.DataFrame() + # for (country, carrier), df in gen.groupby(["country", "carrier"]): + # ins_cap.at[country, carrier] = df.p_nom.sum() + # #ins_cap.drop(columns=["load shedding"], inplace=True) - # Loads - load = etrago.network.loads - load_t = etrago.network.loads_t.p_set - load["country"] = load.bus.map(etrago.network.buses.country) - load_country = pd.DataFrame() - for (country, carrier), df in load.groupby(["country", "carrier"]): - loads_id = df.index - load_country.at[country, carrier] = load_t[loads_id].sum().sum() + # # Loads + # load = etrago.network.loads + # load_t = etrago.network.loads_t.p_set + # load["country"] = load.bus.map(etrago.network.buses.country) + # load_country = pd.DataFrame() + # for (country, carrier), df in load.groupby(["country", "carrier"]): + # loads_id = df.index + # load_country.at[country, carrier] = load_t[loads_id].sum().sum() - # storage_units - sto = etrago.network.storage_units - sto["country"] = sto.bus.map(etrago.network.buses.country) - sto_country = pd.DataFrame() - for (country, carrier), df in sto.groupby(["country", "carrier"]): - sto_country.at[country, carrier] = df.p_nom.sum() + # # storage_units + # sto = etrago.network.storage_units + # sto["country"] = sto.bus.map(etrago.network.buses.country) + # sto_country = pd.DataFrame() + # for (country, carrier), df in sto.groupby(["country", "carrier"]): + # sto_country.at[country, carrier] = df.p_nom.sum() - # links - link = etrago.network.links - link_t = etrago.network.links_t - link_with_t = link.loc[link.index.isin(link_t["p_max_pu"].columns)] + # # links + # link = etrago.network.links + # link_t = etrago.network.links_t + # link_with_t = link.loc[link.index.isin(link_t["p_max_pu"].columns)] - # buses - bus = etrago.network.buses - bus_country = pd.DataFrame() - for (country, carrier), df in bus.groupby(["country", "carrier"]): - bus_country.at[country, carrier] = len(df) + # # buses + # bus = etrago.network.buses + # bus_country = pd.DataFrame() + # for (country, carrier), df in bus.groupby(["country", "carrier"]): + # bus_country.at[country, carrier] = len(df) - # subnetworks - etrago.network.determine_network_topology() - sub_network = etrago.network.buses.loc[etrago.network.buses.carrier=="AC"] - sub_network = sub_network.loc[sub_network.country=="DE"] - sub_network.sub_network.value_counts() - # ehv network clustering - etrago.ehv_clustering() + # # subnetworks + # etrago.network.determine_network_topology() + # sub_network = etrago.network.buses.loc[etrago.network.buses.carrier=="AC"] + # sub_network = sub_network.loc[sub_network.country=="DE"] + # sub_network.sub_network.value_counts() + # # ehv network clustering + # etrago.ehv_clustering() # spatial clustering etrago.spatial_clustering() diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 733b9905a..713905d7d 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -345,7 +345,7 @@ def gas_postprocessing(etrago, busmap, medoid_idx=None): + str(settings["n_clusters_gas"]) + "_result.csv" ) - + breakpoint() busmap = get_h2_clusters(etrago, busmap) # Add all other buses to busmap From 0067605fbf8250afc7d907816d3292c3b85755c2 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Jul 2023 15:13:19 +0200 Subject: [PATCH 44/49] omit H2 clustering when no H2 --- etrago/cluster/gas.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etrago/cluster/gas.py b/etrago/cluster/gas.py index 713905d7d..4385b55a2 100644 --- a/etrago/cluster/gas.py +++ b/etrago/cluster/gas.py @@ -345,8 +345,9 @@ def gas_postprocessing(etrago, busmap, medoid_idx=None): + str(settings["n_clusters_gas"]) + "_result.csv" ) - breakpoint() - busmap = get_h2_clusters(etrago, busmap) + + if 'H2' in etrago.network.buses.carrier.unique(): + busmap = get_h2_clusters(etrago, busmap) # Add all other buses to busmap missing_idx = list( From f8f4c49fcdf8d9e58698c73358e9dce67793785a Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Jul 2023 15:50:36 +0200 Subject: [PATCH 45/49] include status2019 in gas constraints --- etrago/tools/constraints.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index b2d9a0446..a93f8d148 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -1285,6 +1285,12 @@ def read_max_gas_generation(self): """ scn_name = self.args["scn_name"] arg_def = { + ##################### UPDATE VALUE FOR 2019 ########################### + "status2019": { + "CH4": 36000000, + "biogas": 10000000, + }, # [MWh] Netzentwicklungsplan Gas 2020–2030 + ##################### UPDATE VALUE FOR 2019 ########################### "eGon2035": { "CH4": 36000000, "biogas": 10000000, @@ -1297,7 +1303,6 @@ def read_max_gas_generation(self): "biogas": 14450103 }, # [MWh] Value from reference p-e-s run used in eGon-data } - engine = db.connection(section=self.args["db"]) try: sql = f""" @@ -1341,6 +1346,7 @@ def add_ch4_constraints(self, network, snapshots): gas_carrier = arg.keys() carrier_names = { + "status2019": {"CH4": "CH4_NG", "biogas": "CH4_biogas"}, "eGon2035": {"CH4": "CH4_NG", "biogas": "CH4_biogas"}, "eGon2035_lowflex": {"CH4": "CH4_NG", "biogas": "CH4_biogas"}, "eGon100RE": {"biogas": "CH4"}, From 8248da5a1e1a260232ee95f872ea18febe0fe0e1 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Jul 2023 16:05:14 +0200 Subject: [PATCH 46/49] revert commits done for checking process --- etrago/appl.py | 47 +---------------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index a539752fb..95d9242ba 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -50,7 +50,7 @@ args = { # Setup and Configuration: - "db": "powerd-data-server", # database session + "db": "powerd-data", # database session "gridversion": None, # None for model_draft or Version number "method": { # Choose method and settings for optimization "type": "lopf", # type of optimization, currently only 'lopf' @@ -458,51 +458,6 @@ def run_etrago(args, json_path): # adjust network regarding eTraGo setting etrago.adjust_network() - #breakpoint() - - # # Generators - # gen = etrago.network.generators - # gen["country"] = gen.bus.map(etrago.network.buses.country) - # import pandas as pd - # ins_cap = pd.DataFrame() - # for (country, carrier), df in gen.groupby(["country", "carrier"]): - # ins_cap.at[country, carrier] = df.p_nom.sum() - # #ins_cap.drop(columns=["load shedding"], inplace=True) - - # # Loads - # load = etrago.network.loads - # load_t = etrago.network.loads_t.p_set - # load["country"] = load.bus.map(etrago.network.buses.country) - # load_country = pd.DataFrame() - # for (country, carrier), df in load.groupby(["country", "carrier"]): - # loads_id = df.index - # load_country.at[country, carrier] = load_t[loads_id].sum().sum() - - # # storage_units - # sto = etrago.network.storage_units - # sto["country"] = sto.bus.map(etrago.network.buses.country) - # sto_country = pd.DataFrame() - # for (country, carrier), df in sto.groupby(["country", "carrier"]): - # sto_country.at[country, carrier] = df.p_nom.sum() - - # # links - # link = etrago.network.links - # link_t = etrago.network.links_t - # link_with_t = link.loc[link.index.isin(link_t["p_max_pu"].columns)] - - # # buses - # bus = etrago.network.buses - # bus_country = pd.DataFrame() - # for (country, carrier), df in bus.groupby(["country", "carrier"]): - # bus_country.at[country, carrier] = len(df) - - # # subnetworks - # etrago.network.determine_network_topology() - # sub_network = etrago.network.buses.loc[etrago.network.buses.carrier=="AC"] - # sub_network = sub_network.loc[sub_network.country=="DE"] - # sub_network.sub_network.value_counts() - # # ehv network clustering - # etrago.ehv_clustering() # spatial clustering etrago.spatial_clustering() From e919893d42670894854db862b7e452cdd31d649a Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 25 Jul 2023 09:19:00 +0200 Subject: [PATCH 47/49] omit gas constraints for status2019 --- etrago/tools/constraints.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index a93f8d148..82a5037b9 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -1285,12 +1285,6 @@ def read_max_gas_generation(self): """ scn_name = self.args["scn_name"] arg_def = { - ##################### UPDATE VALUE FOR 2019 ########################### - "status2019": { - "CH4": 36000000, - "biogas": 10000000, - }, # [MWh] Netzentwicklungsplan Gas 2020–2030 - ##################### UPDATE VALUE FOR 2019 ########################### "eGon2035": { "CH4": 36000000, "biogas": 10000000, @@ -1346,7 +1340,6 @@ def add_ch4_constraints(self, network, snapshots): gas_carrier = arg.keys() carrier_names = { - "status2019": {"CH4": "CH4_NG", "biogas": "CH4_biogas"}, "eGon2035": {"CH4": "CH4_NG", "biogas": "CH4_biogas"}, "eGon2035_lowflex": {"CH4": "CH4_NG", "biogas": "CH4_biogas"}, "eGon100RE": {"biogas": "CH4"}, @@ -2744,10 +2737,12 @@ def functionality(self, network, snapshots): if "CH4" in network.buses.carrier.values: if self.args["method"]["pyomo"]: add_chp_constraints(network, snapshots) - add_ch4_constraints(self, network, snapshots) + if self.args["scn_name"] != "status2019": + add_ch4_constraints(self, network, snapshots) else: add_chp_constraints_nmp(network) - add_ch4_constraints_nmp(self, network, snapshots) + if self.args["scn_name"] != "status2019": + add_ch4_constraints_nmp(self, network, snapshots) for constraint in self.args["extra_functionality"].keys(): try: From bfc4d0ae2d40dfee804836ffccd4fb0a5fb58cfd Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 10 Aug 2023 14:09:39 +0200 Subject: [PATCH 48/49] restoring app.py and apply black --- etrago/appl.py | 11 +++++++---- etrago/tools/utilities.py | 9 +++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/etrago/appl.py b/etrago/appl.py index 95d9242ba..03e1f0319 100644 --- a/etrago/appl.py +++ b/etrago/appl.py @@ -50,7 +50,7 @@ args = { # Setup and Configuration: - "db": "powerd-data", # database session + "db": "egon-data", # database session "gridversion": None, # None for model_draft or Version number "method": { # Choose method and settings for optimization "type": "lopf", # type of optimization, currently only 'lopf' @@ -74,7 +74,7 @@ "threads": 4, }, "model_formulation": "kirchhoff", # angles or kirchhoff - "scn_name": "status2019", # scenario: eGon2035 or eGon100RE + "scn_name": "eGon2035", # scenario: eGon2035, eGon100RE or status2019 # Scenario variations: "scn_extension": None, # None or array of extension scenarios "scn_decommissioning": None, # None or decommissioning scenario @@ -155,7 +155,7 @@ }, # Simplifications: "branch_capacity_factor": {"HV": 0.5, "eHV": 0.7}, # p.u. branch derating - "load_shedding": False, # meet the demand at value of loss load cost + "load_shedding": True, # meet the demand at value of loss load cost "foreign_lines": { "carrier": "AC", # 'DC' for modeling foreign lines as links "capacity": "osmTGmod", # 'osmTGmod', 'tyndp2020', 'ntc_acer' or 'thermal_acer' @@ -455,10 +455,13 @@ def run_etrago(args, json_path): # import network from database etrago.build_network_from_db() - + # adjust network regarding eTraGo setting etrago.adjust_network() + # ehv network clustering + etrago.ehv_clustering() + # spatial clustering etrago.spatial_clustering() etrago.spatial_clustering_gas() diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 66b37ecd6..6a37c3cfa 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -2336,8 +2336,13 @@ def check_args(etrago): """ - names = ["eGon2035", "eGon100RE", "eGon2035_lowflex", "eGon100RE_lowflex", - "status2019"] + names = [ + "eGon2035", + "eGon100RE", + "eGon2035_lowflex", + "eGon100RE_lowflex", + "status2019", + ] assert ( etrago.args["scn_name"] in names From f31d60706113270879d0e3cf5534e680ebe7ea28 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 7 Sep 2023 18:20:14 +0200 Subject: [PATCH 49/49] Fix KeyError retrieving marginal costs Fixes the following KeyError: KeyError Traceback (most recent call last) Cell In[6], line 1 ----> 1 etrago.adjust_network() File ~/virtualenvs/hyBit/git_repos/eTraGo/etrago/tools/network.py:351, in Etrago.adjust_network(self) 347 self.geolocation_buses() 349 self.load_shedding() --> 351 self.adjust_CH4_gen_carriers() 353 self.set_random_noise(0.01) 355 self.set_q_national_loads(cos_phi=0.9) File ~/virtualenvs/hyBit/git_repos/eTraGo/etrago/tools/utilities.py:2635, in adjust_CH4_gen_carriers(self) 2629 df = pd.read_sql(sql, engine) 2630 # TODO: There might be a bug in here raising a KeyError. 2631 # If you encounter it, that means you have live data 2632 # to test against. Please do a git blame on these 2633 # lines and follow the hints in the commit message to 2634 # fix the bug. -> 2635 marginal_cost = df["marginal_cost"] 2636 except sqlalchemy.exc.ProgrammingError: 2637 marginal_cost = marginal_cost_def File ~/virtualenvs/hyBit/lib/python3.8/site-packages/pandas/core/series.py:981, in Series.getitem(self, key) 978 return self._values[key] 980 elif key_is_scalar: --> 981 return self._get_value(key) 983 if is_hashable(key): 984 # Otherwise index.get_value will raise InvalidIndexError 985 try: 986 # For labels that don't resolve as scalars like tuples and frozensets File ~/virtualenvs/hyBit/lib/python3.8/site-packages/pandas/core/series.py:1089, in Series._get_value(self, label, takeable) 1086 return self._values[label] 1088 # Similar to Index.get_value, but we do not fall back to positional -> 1089 loc = self.index.get_loc(label) 1090 return self.index._get_values_for_loc(self, loc, label) File ~/virtualenvs/hyBit/lib/python3.8/site-packages/pandas/core/indexes/range.py:395, in RangeIndex.get_loc(self, key, method, tolerance) 393 raise KeyError(key) from err 394 self._check_indexing_error(key) --> 395 raise KeyError(key) 396 return super().get_loc(key, method=method, tolerance=tolerance) KeyError: 'marginal_cost' --- etrago/tools/utilities.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 6a37c3cfa..b141b76a9 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -2624,12 +2624,7 @@ def adjust_CH4_gen_carriers(self): FROM scenario.egon_scenario_parameters WHERE name = '{self.args["scn_name"]}';""" df = pd.read_sql(sql, engine) - # TODO: There might be a bug in here raising a `KeyError`. - # If you encounter it, that means you have live data - # to test against. Please do a `git blame` on these - # lines and follow the hints in the commit message to - # fix the bug. - marginal_cost = df["marginal_cost"] + marginal_cost = df["gas_parameters"][0]["marginal_cost"] except sqlalchemy.exc.ProgrammingError: marginal_cost = marginal_cost_def