From 0a17cac0b07f24030a289f11063537e2be041127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 22 Mar 2023 19:08:27 +0100
Subject: [PATCH 01/35] Call `UniformDisaggregation` with named arguments

Doesn't change the behaviour of the code but makes it easier to
understand.
---
 etrago/cluster/disaggregation.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 0b437883..b7531d55 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -733,9 +733,9 @@ def run_disaggregation(self):
                 )
             elif disagg == "uniform":
                 disaggregation = UniformDisaggregation(
-                    self.disaggregated_network,
-                    self.network,
-                    self.clustering,
+                    original_network=self.disaggregated_network,
+                    clustered_network=self.network,
+                    clustering=self.clustering,
                     skip=skip,
                 )
 

From db8232c1b4aca4dc60794860dc4fbaaa79f21dfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 22 Mar 2023 12:16:36 +0100
Subject: [PATCH 02/35] Replace `print`s with proper logging

Use `loguru` for the logging. Get rid of the `"---"` lines in logging,
because log messages should contain some explicit information.
---
 etrago/cluster/disaggregation.py | 37 ++++++++++++++++----------------
 setup.py                         |  1 +
 2 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index b7531d55..0371de29 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -4,6 +4,7 @@
 import cProfile
 import time
 
+from loguru import logger as log
 from pyomo.environ import Constraint
 from pypsa import Network
 import pandas as pd
@@ -275,8 +276,7 @@ def solve(self, scenario, solver):
         }
         profile = cProfile.Profile()
         for i, cluster in enumerate(sorted(clusters)):
-            print("---")
-            print("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n))
+            log.info("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n))
             profile.enable()
             t = time.time()
             partial_network, externals = self.construct_partial_network(
@@ -284,9 +284,9 @@ def solve(self, scenario, solver):
             )
             profile.disable()
             self.stats["clusters"].loc[cluster, "decompose"] = time.time() - t
-            print(
-                "Decomposed in ",
-                self.stats["clusters"].loc[cluster, "decompose"],
+            log.info(
+                "Decomposed in "
+                f'{self.stats["clusters"].loc[cluster, "decompose"]}'
             )
             t = time.time()
             profile.enable()
@@ -295,32 +295,33 @@ def solve(self, scenario, solver):
             )
             profile.disable()
             self.stats["clusters"].loc[cluster, "spread"] = time.time() - t
-            print(
-                "Result distributed in ",
-                self.stats["clusters"].loc[cluster, "spread"],
+            log.info(
+                "Result distributed in "
+                f'{self.stats["clusters"].loc[cluster, "spread"]}'
             )
             profile.enable()
             t = time.time()
             self.transfer_results(partial_network, externals)
             profile.disable()
             self.stats["clusters"].loc[cluster, "transfer"] = time.time() - t
-            print(
-                "Results transferred in ",
-                self.stats["clusters"].loc[cluster, "transfer"],
+            log.info(
+                "Results transferred in "
+                f'{self.stats["clusters"].loc[cluster, "transfer"]}'
             )
 
         profile.enable()
         t = time.time()
-        print("---")
         fs = (mc("sum"), mc("sum"))
         for bt, ts in (
             ("generators", {"p": fs, "q": fs}),
             ("storage_units", {"p": fs, "state_of_charge": fs, "q": fs}),
         ):
-            print("Attribute sums, {}, clustered - disaggregated:".format(bt))
+            log.info(
+                "Attribute sums, {}, clustered - disaggregated:".format(bt)
+            )
             cnb = getattr(self.clustered_network, bt)
             onb = getattr(self.original_network, bt)
-            print(
+            log.info(
                 "{:>{}}: {}".format(
                     "p_nom_opt",
                     4 + len("state_of_charge"),
@@ -329,11 +330,11 @@ def solve(self, scenario, solver):
                 )
             )
 
-            print("Series sums, {}, clustered - disaggregated:".format(bt))
+            log.info("Series sums, {}, clustered - disaggregated:".format(bt))
             cnb = getattr(self.clustered_network, bt + "_t")
             onb = getattr(self.original_network, bt + "_t")
             for s in ts:
-                print(
+                log.info(
                     "{:>{}}: {}".format(
                         s,
                         4 + len("state_of_charge"),
@@ -343,7 +344,7 @@ def solve(self, scenario, solver):
                 )
         profile.disable()
         self.stats["check"] = time.time() - t
-        print("Checks computed in ", self.stats["check"])
+        log.info("Checks computed in {self.stats['check']}")
 
         # profile.print_stats(sort='cumtime')
 
@@ -749,7 +750,7 @@ def run_disaggregation(self):
             self.disaggregated_network.generators_t.q.fillna(0, inplace=True)
 
             self.disaggregated_network.results = self.network.results
-            print(
+            log.info(
                 "Time for overall desaggregation [min]: {:.2}".format(
                     (time.time() - t) / 60
                 )
diff --git a/setup.py b/setup.py
index 4ea9b7ab..c9fd3aaa 100755
--- a/setup.py
+++ b/setup.py
@@ -46,6 +46,7 @@ def read(*names, **kwargs):
         "egoio == 0.4.7",
         "geoalchemy2 >= 0.3.0",
         "geopandas",
+        "loguru",
         "matplotlib >= 3.0.3",
         "oedialect",
         # PyPSA uses a deprecated import that errors with Pyomo 6.4.3.

From 4716abd94daa5b39759ec9088896893aafd05798 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 22 Mar 2023 18:44:14 +0100
Subject: [PATCH 03/35] Log more stuff

---
 etrago/cluster/disaggregation.py | 3 +++
 etrago/tools/utilities.py        | 1 +
 2 files changed, 4 insertions(+)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 0371de29..e9281ced 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -520,6 +520,7 @@ class UniformDisaggregation(Disaggregation):
     def solve_partial_network(
         self, cluster, partial_network, scenario, solver=None
     ):
+        log.debug("Solving partial network.")
         bustypes = {
             "generators": {"group_by": ("carrier",), "series": ("p", "q")},
             "storage_units": {
@@ -541,6 +542,7 @@ def solve_partial_network(
         }
         filters = {"q": lambda o: o.control == "PV"}
         for bustype in bustypes:
+            log.debug(f"Decomposing {bustype}.")
             pn_t = getattr(partial_network, bustype + "_t")
             cl_t = getattr(self.clustered_network, bustype + "_t")
             pn_buses = getattr(partial_network, bustype)
@@ -720,6 +722,7 @@ def update_constraints(network, externals):
 
 
 def run_disaggregation(self):
+    log.debug("Running disaggregation.")
     if self.clustering:
         disagg = self.args.get("disaggregation")
         skip = () if self.args["pf_post_lopf"]["active"] else ("q",)
diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py
index fddaab0e..1524e19d 100755
--- a/etrago/tools/utilities.py
+++ b/etrago/tools/utilities.py
@@ -571,6 +571,7 @@ def load_shedding(self, temporal_disaggregation=False, **kwargs):
     -------
 
     """
+    logger.debug("Shedding the load.")
     if self.args["load_shedding"]:
         if temporal_disaggregation:
             network = self.network_tsa

From d06926b08452903b32885cf4d9e66d3561e06c22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Thu, 23 Mar 2023 18:13:04 +0100
Subject: [PATCH 04/35] Make local variable `busflags` an argument

When closing over local variables, changing the outer variable also
changes the value inside the function. Making the local variable an
argument with default value prevents that and it's also a tiny bit more
efficient.
---
 etrago/cluster/disaggregation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index e9281ced..58deb414 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -77,7 +77,7 @@ def construct_partial_network(self, cluster, scenario):
         # find all lines that have at least one bus inside the cluster
         busflags = self.buses["cluster"] == cluster
 
-        def is_bus_in_cluster(conn):
+        def is_bus_in_cluster(conn, busflags=busflags):
             return busflags[conn]
 
         # Copy configurations to new network

From a8c24ee3164d89dd6bb8771ca060f0160a30a7a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Thu, 23 Mar 2023 18:31:27 +0100
Subject: [PATCH 05/35] Factor `rows` out into a local variable

And put a type annotation on the local variable.
Helps me to figure out what values are getting passed around, i.e. it
makes the code more readable for me. Also, actually makes the code
shorter.
---
 etrago/cluster/disaggregation.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 58deb414..b052ba45 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -93,14 +93,12 @@ def is_bus_in_cluster(conn, busflags=busflags):
 
         line_types = ["lines", "links", "transformers"]
         for line_type in line_types:
+            rows: pd.DataFrame = getattr(self.original_network, line_type)
             # Copy all lines that reside entirely inside the cluster ...
             setattr(
                 partial_network,
                 line_type,
-                filter_internal_connector(
-                    getattr(self.original_network, line_type),
-                    is_bus_in_cluster,
-                ),
+                filter_internal_connector(rows, is_bus_in_cluster),
             )
 
             # ... and their time series
@@ -115,7 +113,7 @@ def is_bus_in_cluster(conn, busflags=busflags):
 
             # Copy all lines whose `bus0` lies within the cluster
             left_external_connectors = filter_left_external_connector(
-                getattr(self.original_network, line_type), is_bus_in_cluster
+                rows, is_bus_in_cluster
             )
 
             def from_busmap(x):
@@ -134,7 +132,7 @@ def from_busmap(x):
 
             # Copy all lines whose `bus1` lies within the cluster
             right_external_connectors = filter_right_external_connector(
-                getattr(self.original_network, line_type), is_bus_in_cluster
+                rows, is_bus_in_cluster
             )
             if not right_external_connectors.empty:
                 ca_option = pd.get_option("mode.chained_assignment")

From 7f61dbe562d5f41b471bc6d3d312bca9976c7651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Thu, 23 Mar 2023 18:38:49 +0100
Subject: [PATCH 06/35] Factor `timeseries` out into a local variable

Same reasons as in the previous commit.
---
 etrago/cluster/disaggregation.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index b052ba45..93d4be02 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -94,6 +94,9 @@ def is_bus_in_cluster(conn, busflags=busflags):
         line_types = ["lines", "links", "transformers"]
         for line_type in line_types:
             rows: pd.DataFrame = getattr(self.original_network, line_type)
+            timeseries: dict[str, pd.DataFrame] = getattr(
+                self.original_network, line_type + "_t"
+            )
             # Copy all lines that reside entirely inside the cluster ...
             setattr(
                 partial_network,
@@ -105,11 +108,7 @@ def is_bus_in_cluster(conn, busflags=busflags):
             # TODO: These are all time series, not just the ones from lines
             #       residing entirely in side the cluster.
             #       Is this a problem?
-            setattr(
-                partial_network,
-                line_type + "_t",
-                getattr(self.original_network, line_type + "_t"),
-            )
+            setattr(partial_network, line_type + "_t", timeseries)
 
             # Copy all lines whose `bus0` lies within the cluster
             left_external_connectors = filter_left_external_connector(

From ff9c359b27c807a89724ea676bf4976469814642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 22 Mar 2023 05:59:58 +0100
Subject: [PATCH 07/35] Use implicit string joining instead of `+`

And move separating spaces to the beginning of the line while at it.
---
 etrago/cluster/disaggregation.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 93d4be02..50095e4f 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -606,9 +606,9 @@ def solve_partial_network(
                         pnb.loc[:, "p_nom_extendable"]
                         == clb.iloc[0].at["p_nom_extendable"]
                     ).all(), (
-                        "The `'p_nom_extendable'` flag for the current "
-                        + "cluster's bus does not have the same value "
-                        + "it has on the buses of it's partial network."
+                        "The `'p_nom_extendable'` flag for the current"
+                        " cluster's bus does not have the same value"
+                        " it has on the buses of it's partial network."
                     )
 
                 if clb.iloc[0].at["p_nom_extendable"]:

From 4848efe77af4cdbbf8ba22416c9d390cd460728e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 00:12:49 +0100
Subject: [PATCH 08/35] Fix typo in comment: "in side" -> "inside"

---
 etrago/cluster/disaggregation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 50095e4f..23a44d67 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -106,7 +106,7 @@ def is_bus_in_cluster(conn, busflags=busflags):
 
             # ... and their time series
             # TODO: These are all time series, not just the ones from lines
-            #       residing entirely in side the cluster.
+            #       residing entirely inside the cluster.
             #       Is this a problem?
             setattr(partial_network, line_type + "_t", timeseries)
 

From aaa04359088142c5cdc8b19ec0e882871da98e54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 00:43:07 +0100
Subject: [PATCH 09/35] Update comment about unfiltered time series

---
 etrago/cluster/disaggregation.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 23a44d67..b98428e3 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -108,6 +108,10 @@ def is_bus_in_cluster(conn, busflags=busflags):
             # TODO: These are all time series, not just the ones from lines
             #       residing entirely inside the cluster.
             #       Is this a problem?
+            #       I hope not, because neither is `rows.index` a subset
+            #       of the columns of one of the values of `timeseries`,
+            #       nor the other way around, so it's not clear how to
+            #       align both.
             setattr(partial_network, line_type + "_t", timeseries)
 
             # Copy all lines whose `bus0` lies within the cluster

From 096f9d3c471807767d669374f6fb4d03e9f5b20f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 00:53:58 +0100
Subject: [PATCH 10/35] Replace `DataFrame.append` with `pandas.concat`

The `DataFrame.append` method is deprecated in favour of `pandas.concat`
as of `pandas` version 1.4.0.
---
 etrago/cluster/disaggregation.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index b98428e3..1904d641 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -181,8 +181,8 @@ def from_busmap(x):
         self.reindex_with_prefix(externals_to_insert)
 
         # .. and insert them as well as their time series
-        partial_network.buses = partial_network.buses.append(
-            externals_to_insert
+        partial_network.buses = pd.concat(
+            [partial_network.buses, externals_to_insert]
         )
         partial_network.buses_t = self.original_network.buses_t
 
@@ -214,7 +214,9 @@ def from_busmap(x):
             setattr(
                 partial_network,
                 bustype,
-                getattr(partial_network, bustype).append(buses_to_insert),
+                pd.concat(
+                    [getattr(partial_network, bustype), buses_to_insert]
+                ),
             )
 
             # Also copy their time series

From 6ae2a45fb8856348b7a2629afca7329bf430cbdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 01:15:14 +0100
Subject: [PATCH 11/35] Improve sanity check's comprehensibility

Use descriptive variable names for intermediate values to make the code
shorter and basically read as what it does: assert that the `DataFrame`
of rows which don't pass the sanity check, i.e. are "not sane", or
`~sane`, is empty.
---
 etrago/cluster/disaggregation.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 1904d641..b976b123 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -246,16 +246,13 @@ def from_busmap(x):
         # Just a simple sanity check
         # TODO: Remove when sure that disaggregation will not go insane anymore
         for line_type in line_types:
-            assert (
-                getattr(partial_network, line_type)
-                .bus0.isin(partial_network.buses.index)
-                .all()
-            )
-            assert (
-                getattr(partial_network, line_type)
-                .bus1.isin(partial_network.buses.index)
-                .all()
-            )
+            rows = getattr(partial_network, line_type)
+
+            sane = rows.bus0.isin(partial_network.buses.index)
+            assert rows.loc[~sane, :].empty
+
+            sane = rows.bus1.isin(partial_network.buses.index)
+            assert rows.loc[~sane, :].empty
 
         return partial_network, external_buses
 

From 4339ebedd22cb038518af0ed3009073982cb9c20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 01:23:51 +0100
Subject: [PATCH 12/35] Add an error message to sanity checks

---
 etrago/cluster/disaggregation.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index b976b123..4fe0b798 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -249,10 +249,18 @@ def from_busmap(x):
             rows = getattr(partial_network, line_type)
 
             sane = rows.bus0.isin(partial_network.buses.index)
-            assert rows.loc[~sane, :].empty
+            assert rows.loc[~sane, :].empty, (
+                f"Not all `partial_network.{line_type}.bus0` entries are"
+                f" contained in `partial_network.buses.index`."
+                f" Spurious additional rows:\nf{rows.loc[~sane, :]}"
+            )
 
             sane = rows.bus1.isin(partial_network.buses.index)
-            assert rows.loc[~sane, :].empty
+            assert rows.loc[~sane, :].empty, (
+                f"Not all `partial_network.{line_type}.bus1` entries are"
+                f" contained in `partial_network.buses.index`."
+                f" Spurious additional rows:\nf{rows.loc[~sane, :]}"
+            )
 
         return partial_network, external_buses
 

From 021d83bb99b73c1dc71655f9d5233ae68943e0b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 01:26:33 +0100
Subject: [PATCH 13/35] Bulk insert disaggregated columns

Instead of using a for loop to append them one at a time. Doing so
triggered a few warnings about the `DataFrame` being fragmented. The new
code should prevent that, while still inserting the same values.
Additionally, no duplicate columns are created. Instead the data in
already existing columns is overwritten, which is a plus.
---
 etrago/cluster/disaggregation.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 4fe0b798..4d6deb4f 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -680,9 +680,13 @@ def solve_partial_network(
                         else ()
                     )
                     ws = weight.sum(axis=len(loc))
-                    for bus_id in filtered.index:
-                        values = clt * weight.loc[loc + (bus_id,)] / ws
-                        pn_t[s].insert(len(pn_t[s].columns), bus_id, values)
+                    new_columns = pd.DataFrame(
+                        {
+                            bus_id: clt * weight.loc[loc + (bus_id,)] / ws
+                            for bus_id in filtered.index
+                        }
+                    )
+                    pn_t[s].loc[:, new_columns.columns] = new_columns
 
     def transfer_results(self, *args, **kwargs):
         kwargs["bustypes"] = ["generators", "storage_units"]

From f00d49aa9c6dd0a89364cc30fd541c2ef257d387 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 01:45:17 +0100
Subject: [PATCH 14/35] Add some information to "etrago/tools/__init__.py"

A short docstring along with author, copyright and license information.
---
 etrago/tools/__init__.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/etrago/tools/__init__.py b/etrago/tools/__init__.py
index 497ec4c5..df40335b 100644
--- a/etrago/tools/__init__.py
+++ b/etrago/tools/__init__.py
@@ -1,8 +1,12 @@
-"""
+"""Multi purpose tools that don't fit anywhere else in eTraGo.
 """
 
-__copyright__ = "tba"
-__license__ = "tba"
-__author__ = "tba"
+__copyright__ = (
+    "Copyright (C) 2023"
+    " Otto-von-Guericke-University Magdeburg,"
+    " Research group for theoretical computer science"
+)
+__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__author__ = "gnn <gnn.code@gmail.com>"
 
 

From 4ecd568dfc0de05d1b8a9832f1c63a9864708c19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 03:52:00 +0100
Subject: [PATCH 15/35] Add various ways of doing nothing

---
 etrago/tools/__init__.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/etrago/tools/__init__.py b/etrago/tools/__init__.py
index df40335b..170b8714 100644
--- a/etrago/tools/__init__.py
+++ b/etrago/tools/__init__.py
@@ -10,3 +10,29 @@
 __author__ = "gnn <gnn.code@gmail.com>"
 
 
+def noop(*ignored_arguments, **ignored_keyword_arguments):
+    """Do nothing.
+
+    Accept all kinds of arguments, ignore them and do nothing.
+    """
+    pass
+
+
+class Noops:
+    """Provide arbitrarily named methods that do nothing.
+
+    Any attribute access will return a method that does nothing, i.e.
+    all methods of this object are :py:func:`noop`s. Normally you don't
+    need to instantiate this class. All instances behave the same, so
+    the containing module provides one called :py:obj:`noops` which you
+    can import and use.
+    """
+
+    @classmethod
+    def __getattribute__(cls, ignored_name):
+        return noop
+
+
+noops = Noops()
+"""A default :py:class:`Noops` instance so you don't have to create one.
+"""

From 17c5e631b6f11d3f05efdc0eb69cd43c6fbfac1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 03:56:41 +0100
Subject: [PATCH 16/35] Disable profiling more thorough

Using `noops` not only disables the profiling report, but also makes
`profile.enable()` and `profile.disable()` calls do nothing with as
little overhead as possible.
---
 etrago/cluster/disaggregation.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 4d6deb4f..cd0ef299 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -9,6 +9,7 @@
 from pypsa import Network
 import pandas as pd
 
+from etrago.tools import noops
 from etrago.tools.utilities import residual_load
 
 
@@ -283,6 +284,7 @@ def solve(self, scenario, solver):
             )
         }
         profile = cProfile.Profile()
+        profile = noops
         for i, cluster in enumerate(sorted(clusters)):
             log.info("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n))
             profile.enable()
@@ -354,7 +356,7 @@ def solve(self, scenario, solver):
         self.stats["check"] = time.time() - t
         log.info("Checks computed in {self.stats['check']}")
 
-        # profile.print_stats(sort='cumtime')
+        profile.print_stats(sort="cumtime")
 
     def transfer_results(
         self,

From c350ae4f7b26e9c09ddd243cff4fe713cb399092 Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Thu, 12 Jan 2023 09:53:08 +0100
Subject: [PATCH 17/35] Don't create `disaggregated_network.results`

Results are stored directly in `disaggregated_network`'s attributes.
---
 etrago/cluster/disaggregation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index cd0ef299..ecfa7f8d 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -766,7 +766,6 @@ def run_disaggregation(self):
             self.disaggregated_network.generators_t.p.fillna(0, inplace=True)
             self.disaggregated_network.generators_t.q.fillna(0, inplace=True)
 
-            self.disaggregated_network.results = self.network.results
             log.info(
                 "Time for overall desaggregation [min]: {:.2}".format(
                     (time.time() - t) / 60

From 9db0c2d67163ad99a65119f74b0d89c5a4806c9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 3 Mar 2023 16:51:19 +0100
Subject: [PATCH 18/35] Use a shorter way of getting the first index value

That's what `next(clb.itertuples()).Index` does. It just gets the index
of the first row of `clb`, so `clb.index[0]` should be equivalent but
way clearer and more readable.
Also note that `clb` should only have one row anyway.
---
 etrago/cluster/disaggregation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index ecfa7f8d..2f45e9d0 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -665,7 +665,7 @@ def solve_partial_network(
                     if s in self.skip:
                         continue
                     filtered = pnb.loc[filters.get(s, slice(None))]
-                    clt = cl_t[s].loc[:, next(clb.itertuples()).Index]
+                    clt = cl_t[s].loc[:, clb.index[0]]
                     weight = reduce(
                         multiply,
                         (

From 57352b106e6b193f98be44c8935003b2ca00f7ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 22 Mar 2023 06:04:34 +0100
Subject: [PATCH 19/35] Replace older string interpolation with f-strings

---
 etrago/cluster/disaggregation.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 2f45e9d0..79e61414 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -286,7 +286,7 @@ def solve(self, scenario, solver):
         profile = cProfile.Profile()
         profile = noops
         for i, cluster in enumerate(sorted(clusters)):
-            log.info("Decompose cluster %s (%d/%d)" % (cluster, i + 1, n))
+            log.info(f"Decompose {cluster=} ({i + 1}/{n})")
             profile.enable()
             t = time.time()
             partial_network, externals = self.construct_partial_network(
@@ -326,9 +326,7 @@ def solve(self, scenario, solver):
             ("generators", {"p": fs, "q": fs}),
             ("storage_units", {"p": fs, "state_of_charge": fs, "q": fs}),
         ):
-            log.info(
-                "Attribute sums, {}, clustered - disaggregated:".format(bt)
-            )
+            log.info(f"Attribute sums, {bt}, clustered - disaggregated:")
             cnb = getattr(self.clustered_network, bt)
             onb = getattr(self.original_network, bt)
             log.info(
@@ -340,7 +338,7 @@ def solve(self, scenario, solver):
                 )
             )
 
-            log.info("Series sums, {}, clustered - disaggregated:".format(bt))
+            log.info(f"Series sums, {bt}, clustered - disaggregated:")
             cnb = getattr(self.clustered_network, bt + "_t")
             onb = getattr(self.original_network, bt + "_t")
             for s in ts:
@@ -575,10 +573,8 @@ def solve_partial_network(
                 if len(clb) == 0:
                     continue
                 assert len(clb) == 1, (
-                    "Cluster {} has {} buses for group {}.\n".format(
-                        cluster, len(clb), group
-                    )
-                    + "Should be exactly one."
+                    f"Cluster {cluster} has {len(clb)} buses for {group=}."
+                    "\nShould be exactly one."
                 )
                 # Remove buses not belonging to the partial network
                 pnb = pn_buses.iloc[

From 988d19cb95e8811d4e6e425f89952d58eb992468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 24 Mar 2023 05:33:50 +0100
Subject: [PATCH 20/35] Disaggregate "p0" and "p1" of "links"

Instead of only retaining connecting components, i.e. "lines", "links"
and "transformers", which are completely contained in a cluster, retain
all components which have at least one endpoint inside the cluster. This
is what the switch from `&` to `|` does. That way, these components are
available as `cl_buses` when running `solve_partial_network`. Then, when
solving the partial network, only select those components for
disaggregation which have the left endpoint, i.e. `bus0` in the cluster
and group them by right endpoint, i.e. `bus1`. Adjust sanity checks
accordingly, as now a component doesn't have to have both of its
endpoints inside the cluster, but only needs to have one.
---
 etrago/cluster/disaggregation.py | 36 +++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 79e61414..6d5a89b5 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -249,18 +249,13 @@ def from_busmap(x):
         for line_type in line_types:
             rows = getattr(partial_network, line_type)
 
-            sane = rows.bus0.isin(partial_network.buses.index)
-            assert rows.loc[~sane, :].empty, (
-                f"Not all `partial_network.{line_type}.bus0` entries are"
+            left = rows.bus0.isin(partial_network.buses.index)
+            right = rows.bus1.isin(partial_network.buses.index)
+            assert rows.loc[~(left | right), :].empty, (
+                f"Not all `partial_network.{line_type}` have an endpoint,"
+                " i.e. `bus0` or `bus1`,"
                 f" contained in `partial_network.buses.index`."
-                f" Spurious additional rows:\nf{rows.loc[~sane, :]}"
-            )
-
-            sane = rows.bus1.isin(partial_network.buses.index)
-            assert rows.loc[~sane, :].empty, (
-                f"Not all `partial_network.{line_type}.bus1` entries are"
-                f" contained in `partial_network.buses.index`."
-                f" Spurious additional rows:\nf{rows.loc[~sane, :]}"
+                f" Spurious additional rows:\nf{rows.loc[~(left | right), :]}"
             )
 
         return partial_network, external_buses
@@ -530,6 +525,10 @@ def solve_partial_network(
     ):
         log.debug("Solving partial network.")
         bustypes = {
+            "links": {
+                "group_by": ("carrier", "bus1"),
+                "series": ("p0", "p1"),
+            },
             "generators": {"group_by": ("carrier",), "series": ("p", "q")},
             "storage_units": {
                 "group_by": ("carrier", "max_hours"),
@@ -546,6 +545,8 @@ def solve_partial_network(
                 )
                 else ("p_nom_opt", "p_max_pu")
             ),
+            "p0": ("p_nom_opt",),
+            "p1": ("p_nom_opt",),
             "state_of_charge": ("p_nom_opt",),
         }
         filters = {"q": lambda o: o.control == "PV"}
@@ -565,7 +566,11 @@ def solve_partial_network(
                 ]
             )
             for group in groups:
-                clb = cl_buses[cl_buses.bus == cluster]
+                clb = (
+                    cl_buses[cl_buses.bus == cluster]
+                    if "bus" in cl_buses.columns
+                    else cl_buses[cl_buses.bus0 == cluster]
+                )
                 query = " & ".join(
                     ["({key} == {value!r})".format(**axis) for axis in group]
                 )
@@ -581,7 +586,10 @@ def solve_partial_network(
                     [
                         i
                         for i, row in enumerate(pn_buses.itertuples())
-                        if not row.bus.startswith(self.idx_prefix)
+                        for bus in [
+                            row.bus if hasattr(row, "bus") else row.bus1
+                        ]
+                        if not bus.startswith(self.idx_prefix)
                     ]
                 ]
                 pnb = pnb.query(query)
@@ -701,7 +709,7 @@ def swap_series(s):
 
 def filter_internal_connector(conn, is_bus_in_cluster):
     return conn[
-        conn.bus0.apply(is_bus_in_cluster) & conn.bus1.apply(is_bus_in_cluster)
+        conn.bus0.apply(is_bus_in_cluster) | conn.bus1.apply(is_bus_in_cluster)
     ]
 
 

From b6f42cd6b4f7f2e6fd343fb7d9537a9d908bcba2 Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Wed, 11 Jan 2023 18:33:23 +0100
Subject: [PATCH 21/35] Disaggregate "stores"

Differentiate between the different columns of stores and other
components, as stores use "e_.*" where other components use "p_.*".
---
 etrago/cluster/disaggregation.py | 49 +++++++++++++++++++++++---------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 6d5a89b5..fe856b3a 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -534,6 +534,10 @@ def solve_partial_network(
                 "group_by": ("carrier", "max_hours"),
                 "series": ("p", "state_of_charge", "q"),
             },
+            "stores": {
+                "group_by": ("carrier",),
+                "series": ("e", "p"),
+            },
         }
         weights = {
             "p": ("p_nom_opt", "p_max_pu"),
@@ -548,14 +552,31 @@ def solve_partial_network(
             "p0": ("p_nom_opt",),
             "p1": ("p_nom_opt",),
             "state_of_charge": ("p_nom_opt",),
+            "e": ("e_nom_opt",),
         }
         filters = {"q": lambda o: o.control == "PV"}
+
         for bustype in bustypes:
+            # Define attributeof components which are available
+            if bustype == "stores":
+                extendable_flag = "e_nom_extendable"
+                nominal_capacity = "e_nom"
+                optimal_capacity = "e_nom_opt"
+                maximal_capacity = "e_nom_max"
+                weights["p"] = ("e_nom_opt", "e_max_pu")
+            else:
+                extendable_flag = "p_nom_extendable"
+                nominal_capacity = "p_nom"
+                optimal_capacity = "p_nom_opt"
+                maximal_capacity = "p_nom_max"
+                weights["p"] = ("p_nom_opt", "p_max_pu")
+
             log.debug(f"Decomposing {bustype}.")
             pn_t = getattr(partial_network, bustype + "_t")
             cl_t = getattr(self.clustered_network, bustype + "_t")
             pn_buses = getattr(partial_network, bustype)
             cl_buses = getattr(self.clustered_network, bustype)
+
             groups = product(
                 *[
                     [
@@ -603,8 +624,8 @@ def solve_partial_network(
                 )
 
                 if not (
-                    pnb.loc[:, "p_nom_extendable"].all()
-                    or not pnb.loc[:, "p_nom_extendable"].any()
+                    pnb.loc[:, extendable_flag].all()
+                    or not pnb.loc[:, extendable_flag].any()
                 ):
                     raise NotImplementedError(
                         "The `'p_nom_extendable'` flag for buses in the"
@@ -620,41 +641,41 @@ def solve_partial_network(
                     )
                 else:
                     assert (
-                        pnb.loc[:, "p_nom_extendable"]
-                        == clb.iloc[0].at["p_nom_extendable"]
+                        pnb.loc[:, extendable_flag]
+                        == clb.iloc[0].at[extendable_flag]
                     ).all(), (
                         "The `'p_nom_extendable'` flag for the current"
                         " cluster's bus does not have the same value"
                         " it has on the buses of it's partial network."
                     )
 
-                if clb.iloc[0].at["p_nom_extendable"]:
+                if clb.iloc[0].at[extendable_flag]:
                     # That means, `p_nom` got computed via optimization and we
                     # have to distribute it into the subnetwork first.
-                    pnb_p_nom_max = pnb.loc[:, "p_nom_max"]
+                    pnb_p_nom_max = pnb.loc[:, maximal_capacity]
                     p_nom_max_global = pnb_p_nom_max.sum(axis="index")
-                    pnb.loc[:, "p_nom_opt"] = (
-                        clb.iloc[0].at["p_nom_opt"]
+                    pnb.loc[:, optimal_capacity] = (
+                        clb.iloc[0].at[optimal_capacity]
                         * pnb_p_nom_max
                         / p_nom_max_global
                     )
                     getattr(self.original_network, bustype).loc[
-                        pnb.index, "p_nom_opt"
-                    ] = pnb.loc[:, "p_nom_opt"]
-                    pnb.loc[:, "p_nom"] = pnb.loc[:, "p_nom_opt"]
+                        pnb.index, optimal_capacity
+                    ] = pnb.loc[:, optimal_capacity]
+                    pnb.loc[:, nominal_capacity] = pnb.loc[:, optimal_capacity]
                 else:
                     # That means 'p_nom_opt' didn't get computed and is
                     # potentially not present in the dataframe. But we want to
                     # always use 'p_nom_opt' in the remaining code, so save a
                     # view of the computed 'p_nom' values under 'p_nom_opt'.
-                    pnb.loc[:, "p_nom_opt"] = pnb.loc[:, "p_nom"]
+                    pnb.loc[:, optimal_capacity] = pnb.loc[:, nominal_capacity]
 
                 # This probably shouldn't be here, but rather in
                 # `transfer_results`, but it's easier to do it this way right
                 # now.
                 getattr(self.original_network, bustype).loc[
-                    pnb.index, "p_nom_opt"
-                ] = pnb.loc[:, "p_nom_opt"]
+                    pnb.index, optimal_capacity
+                ] = pnb.loc[:, optimal_capacity]
                 timed = (
                     lambda key, series=set(
                         s

From f109cec35b0e4713564ab0ebcf960e98b79b4bf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Mon, 16 Jan 2023 17:20:48 +0100
Subject: [PATCH 22/35] Transfer `"links"`' and `"stores"`' results

---
 etrago/cluster/disaggregation.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index fe856b3a..f41453b3 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -716,10 +716,12 @@ def solve_partial_network(
                     pn_t[s].loc[:, new_columns.columns] = new_columns
 
     def transfer_results(self, *args, **kwargs):
-        kwargs["bustypes"] = ["generators", "storage_units"]
+        kwargs["bustypes"] = ["generators", "links", "storage_units", "stores"]
         kwargs["series"] = {
             "generators": {"p"},
+            "links": {"p0", "p1"},
             "storage_units": {"p", "state_of_charge"},
+            "stores": {"e", "p"},
         }
         return super().transfer_results(*args, **kwargs)
 

From 886211bf49cc34b96108ec63bc0c40b455b66adc Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Mon, 6 Feb 2023 14:52:14 +0100
Subject: [PATCH 23/35] Replace infinite values with a large constant

Use a large constant to avoid NaNs as disaggregation weights.
---
 etrago/cluster/disaggregation.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index f41453b3..78f5cc75 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -653,7 +653,13 @@ def solve_partial_network(
                     # That means, `p_nom` got computed via optimization and we
                     # have to distribute it into the subnetwork first.
                     pnb_p_nom_max = pnb.loc[:, maximal_capacity]
+
+                    # If upper limit is infinite, replace it by a very large
+                    # number to avoid NaN values in the calculation
+                    pnb_p_nom_max.replace(float("inf"), 10000000, inplace=True)
+
                     p_nom_max_global = pnb_p_nom_max.sum(axis="index")
+
                     pnb.loc[:, optimal_capacity] = (
                         clb.iloc[0].at[optimal_capacity]
                         * pnb_p_nom_max

From 854efd9814d7e028ee9d42984d40ca5b4b1006f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Wed, 8 Mar 2023 16:36:14 +0100
Subject: [PATCH 24/35] Reformat the definition of `timed`

Just putting `key in series` in parenthesis prevents `black` from
spreading it over multiple lines, which means that the whole `lambda`
expression doesn't have to be put in parenthesis, making the whole thing
less indented and a bit more readable IMHO.
Also note that it uses `{}` instead of `set()` now.
Last but not least, use a `# noqa` comment do pacify linters complaining
about an assigned lambda. A `def` would make these lines less compact
and thus slightly less readable, IMHO.
---
 etrago/cluster/disaggregation.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 78f5cc75..2c18601c 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -682,15 +682,12 @@ def solve_partial_network(
                 getattr(self.original_network, bustype).loc[
                     pnb.index, optimal_capacity
                 ] = pnb.loc[:, optimal_capacity]
-                timed = (
-                    lambda key, series=set(
-                        s
-                        for s in cl_t
-                        if not cl_t[s].empty
-                        if not pn_t[s].columns.intersection(pnb.index).empty
-                    ): key
-                    in series
-                )
+                timed = lambda key, series={  # noqa: 731
+                    s
+                    for s in cl_t
+                    if not cl_t[s].empty
+                    if not pn_t[s].columns.intersection(pnb.index).empty
+                }: (key in series)
 
                 for s in bustypes[bustype]["series"]:
                     if s in self.skip:

From 73b9e3b356ef1a6df923204d424cc7a9792e6e46 Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Mon, 27 Mar 2023 09:39:52 +0200
Subject: [PATCH 25/35] Add busmap as type pandas.Series

---
 etrago/cluster/disaggregation.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 2c18601c..c727fc47 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -26,10 +26,11 @@ def __init__(
         self.original_network = original_network
         self.clustered_network = clustered_network
         self.clustering = clustering
+        self.busmap = pd.Series(clustering.busmap)
 
         self.buses = pd.merge(
             original_network.buses,
-            clustering.busmap.to_frame(name="cluster"),
+            self.busmap.to_frame(name="cluster"),
             left_index=True,
             right_index=True,
         )
@@ -121,7 +122,7 @@ def is_bus_in_cluster(conn, busflags=busflags):
             )
 
             def from_busmap(x):
-                return self.idx_prefix + self.clustering.busmap.loc[x]
+                return self.idx_prefix + self.busmap.loc[x]
 
             if not left_external_connectors.empty:
                 ca_option = pd.get_option("mode.chained_assignment")
@@ -270,7 +271,7 @@ def solve(self, scenario, solver):
         :param scenario:
         :param solver: Solver that may be used to optimize partial networks
         """
-        clusters = set(self.clustering.busmap.values)
+        clusters = set(self.busmap.values)
         n = len(clusters)
         self.stats = {
             "clusters": pd.DataFrame(
@@ -424,7 +425,7 @@ def _validate_disaggregation_generators(self, cluster, f):
         def extra_functionality(network, snapshots):
             f(network, snapshots)
             generators = self.original_network.generators.assign(
-                bus=lambda df: df.bus.map(self.clustering.busmap)
+                bus=lambda df: df.bus.map(self.busmap)
             )
 
             def construct_constraint(model, snapshot, carrier):
@@ -476,7 +477,7 @@ def extra_functionality(network, snapshots):
             ]:
                 generators = getattr(
                     self.original_network, bustype_pypsa
-                ).assign(bus=lambda df: df.bus.map(self.clustering.busmap))
+                ).assign(bus=lambda df: df.bus.map(self.busmap))
                 for suffix in suffixes:
 
                     def construct_constraint(model, snapshot):

From d806f8b556cd082d18097aaa0d51a213e2561d4f Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Mon, 27 Mar 2023 09:40:29 +0200
Subject: [PATCH 26/35] Un-comment spatial disaggregation

---
 etrago/appl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etrago/appl.py b/etrago/appl.py
index da10a4e3..ee851d3a 100644
--- a/etrago/appl.py
+++ b/etrago/appl.py
@@ -134,7 +134,7 @@
         },
     },
     "network_clustering_ehv": False,  # clustering of HV buses to EHV buses.
-    "disaggregation": None,  # None, 'mini' or 'uniform'
+    "disaggregation": "uniform",  # None, 'mini' or 'uniform'
     # Temporal Complexity:
     "snapshot_clustering": {
         "active": False,  # choose if clustering is activated
@@ -517,7 +517,7 @@ def run_etrago(args, json_path):
 
     # spatial disaggregation
     # needs to be adjusted for new sectors
-    # etrago.disaggregation()
+    etrago.disaggregation()
 
     # calculate central etrago results
     etrago.calc_results()

From 95c22f016fba3dfb22074fa7b5c0c7832a089482 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Thu, 20 Apr 2023 17:09:42 +0200
Subject: [PATCH 27/35] Replace `.busmap` with `.buses.loc[:, "cluster"]`

This partially reverts commit 73b9e3b3. The `.busmap` attribute isn't
really necessary, as `.buses.loc[:, "cluster"]` returns the same
`Series`, i.e. the same indices, values and mapping of index to value.
The only difference is in the order of the entries, which shouldn't
matter.
---
 etrago/cluster/disaggregation.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index c727fc47..296977aa 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -26,11 +26,10 @@ def __init__(
         self.original_network = original_network
         self.clustered_network = clustered_network
         self.clustering = clustering
-        self.busmap = pd.Series(clustering.busmap)
 
         self.buses = pd.merge(
             original_network.buses,
-            self.busmap.to_frame(name="cluster"),
+            self.clustering.busmap.to_frame(name="cluster"),
             left_index=True,
             right_index=True,
         )
@@ -122,7 +121,7 @@ def is_bus_in_cluster(conn, busflags=busflags):
             )
 
             def from_busmap(x):
-                return self.idx_prefix + self.busmap.loc[x]
+                return self.idx_prefix + self.buses.loc[x, "cluster"]
 
             if not left_external_connectors.empty:
                 ca_option = pd.get_option("mode.chained_assignment")
@@ -271,7 +270,7 @@ def solve(self, scenario, solver):
         :param scenario:
         :param solver: Solver that may be used to optimize partial networks
         """
-        clusters = set(self.busmap.values)
+        clusters = set(self.buses.loc[:, "cluster"].values)
         n = len(clusters)
         self.stats = {
             "clusters": pd.DataFrame(
@@ -425,7 +424,7 @@ def _validate_disaggregation_generators(self, cluster, f):
         def extra_functionality(network, snapshots):
             f(network, snapshots)
             generators = self.original_network.generators.assign(
-                bus=lambda df: df.bus.map(self.busmap)
+                bus=lambda df: df.bus.map(self.buses.loc[:, "cluster"])
             )
 
             def construct_constraint(model, snapshot, carrier):
@@ -477,7 +476,9 @@ def extra_functionality(network, snapshots):
             ]:
                 generators = getattr(
                     self.original_network, bustype_pypsa
-                ).assign(bus=lambda df: df.bus.map(self.busmap))
+                ).assign(
+                    bus=lambda df: df.bus.map(self.buses.loc[:, "cluster"])
+                )
                 for suffix in suffixes:
 
                     def construct_constraint(model, snapshot):

From 0a9c5156358c267728798f2f4c8c5734a396993f Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Tue, 18 Apr 2023 17:47:26 +0200
Subject: [PATCH 28/35] Filter links by `bus0` instead of `bus1`

This fixes a bug in the original `p0`, `p1` disaggregation
implementation for `links` in commit 988d19cb9. For components having a
`bus0` and a `bus1` instead of just a `bus`, `bus0` is taken as
corresponding to `bus`. So filtering out these components should be done
by looking at `bus0`, not `bus1`.
---
 etrago/cluster/disaggregation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 296977aa..67b77522 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -610,7 +610,7 @@ def solve_partial_network(
                         i
                         for i, row in enumerate(pn_buses.itertuples())
                         for bus in [
-                            row.bus if hasattr(row, "bus") else row.bus1
+                            row.bus if hasattr(row, "bus") else row.bus0
                         ]
                         if not bus.startswith(self.idx_prefix)
                     ]

From 7a5a100eac7b1464a07aa555893f23dd67330ceb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 21 Apr 2023 06:18:14 +0200
Subject: [PATCH 29/35] Move `bus == cluster` filtering out of the loop

Nothing in the filter uses the loop variable, so the filtering can be
moved out of the loop. Also, since `cl_buses` is only used to determine
the value of `clb` for the duration of the loop and nowhere else,
filtering can be done directly when assigning `cl_buses` and `clb` can
be determined directly by querying the filtered DataFrame.
Filtering before determining the groups massively speeds up the
disaggregation.
---
 etrago/cluster/disaggregation.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 67b77522..06d04431 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -577,8 +577,10 @@ def solve_partial_network(
             pn_t = getattr(partial_network, bustype + "_t")
             cl_t = getattr(self.clustered_network, bustype + "_t")
             pn_buses = getattr(partial_network, bustype)
-            cl_buses = getattr(self.clustered_network, bustype)
-
+            cl_buses = getattr(self.clustered_network, bustype)[
+                lambda df: df.loc[:, "bus" if "bus" in df.columns else "bus0"]
+                == cluster
+            ]
             groups = product(
                 *[
                     [
@@ -589,15 +591,10 @@ def solve_partial_network(
                 ]
             )
             for group in groups:
-                clb = (
-                    cl_buses[cl_buses.bus == cluster]
-                    if "bus" in cl_buses.columns
-                    else cl_buses[cl_buses.bus0 == cluster]
-                )
                 query = " & ".join(
                     ["({key} == {value!r})".format(**axis) for axis in group]
                 )
-                clb = clb.query(query)
+                clb = cl_buses.query(query)
                 if len(clb) == 0:
                     continue
                 assert len(clb) == 1, (

From a17f05dface0fc9eeb27f99c98179818b1b9bbac Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Tue, 18 Apr 2023 17:45:17 +0200
Subject: [PATCH 30/35] Adjust `query` for the special case of `"links"`

For links, the `query` contains references to `bus1`, which has a
different meaning depending on whether we are dealing with a cluster or
the partial network the cluster represents. For clusters, the query is
correct, because the group was generated by looking up the `bus1` values
of links attached to the cluster. But to find the corresponding links in
the partial network, one has to look for those links whose `bus1` value
matches the index of one of the buses whose `"cluster"` entry is equal
to the group's `bus1` value.
Or in other words, since the group's `bus1` value identifies a cluster,
one has to

  - first find all buses corresponding to that cluster,
  - then find all links whose `bus1` value points to one of these buses.
---
 etrago/cluster/disaggregation.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 06d04431..45e08909 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -612,6 +612,14 @@ def solve_partial_network(
                         if not bus.startswith(self.idx_prefix)
                     ]
                 ]
+                if bustype == "links":
+                    index = self.buses[
+                        self.buses.loc[:, "cluster"] == group[1]["value"]
+                    ].index.tolist()
+                    query = (
+                        f"(carrier == {group[0]['value']!r})"
+                        f" & (bus1 in {index})"
+                    )
                 pnb = pnb.query(query)
                 assert not pnb.empty, (
                     "Cluster has a bus for:"

From e579d20dec8004fea643328a9fd7417aa17b5659 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 21 Apr 2023 06:49:59 +0200
Subject: [PATCH 31/35] Use values in cluster columns to define groups

This is the correct way anyway and fixes a bug, since the values stored
in each group (should) uniquely identify a cluster. The bug just never
surfaced, as these values are usually also found in the partial network.
This isn't the case for the values of `bus`, `bus0` and `bus1` though,
because these point to other cluster or partial network buses, so they
differ depending on whether the attribute is on a cluster or a partial
network component, so the bug was finally caught.
---
 etrago/cluster/disaggregation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 45e08909..abb6f366 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -585,7 +585,7 @@ def solve_partial_network(
                 *[
                     [
                         {"key": key, "value": value}
-                        for value in set(pn_buses.loc[:, key])
+                        for value in set(cl_buses.loc[:, key])
                     ]
                     for key in bustypes[bustype]["group_by"]
                 ]

From 12a1933d996e33a713fd12d7d4d0a218eeadfb7f Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Tue, 18 Apr 2023 17:46:48 +0200
Subject: [PATCH 32/35] Allow `pnb` to be empty in certain cases

---
 etrago/cluster/disaggregation.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index abb6f366..a4144ae8 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -621,7 +621,23 @@ def solve_partial_network(
                         f" & (bus1 in {index})"
                     )
                 pnb = pnb.query(query)
-                assert not pnb.empty, (
+                assert not pnb.empty or (
+                    # In some cases, a district heating grid is connected to a
+                    # substation only via a resistive_heater but not e.g. by a
+                    # heat_pump or one of the other listed `carrier`s.
+                    # In the clustered network, there are both.
+                    # In these cases, the `pnb` can actually be empty.
+                    group[0]["value"]
+                    in [
+                        "central_gas_boiler",
+                        "central_heat_pump",
+                        "central_gas_CHP_heat",
+                        "central_gas_CHP",
+                        "CH4",
+                        "DC",
+                        "OCGT",
+                    ]
+                ), (
                     "Cluster has a bus for:"
                     + "\n    ".join(
                         ["{key}: {value!r}".format(**axis) for axis in group]
@@ -629,6 +645,8 @@ def solve_partial_network(
                     + "\nbut no matching buses in its corresponding "
                     + "partial network."
                 )
+                if pnb.empty:
+                    continue
 
                 if not (
                     pnb.loc[:, extendable_flag].all()

From 774ee1a7f4ec192fd7eeb55a9ea071b8b0bef956 Mon Sep 17 00:00:00 2001
From: ClaraBuettner <clara.buettner@hs-flensburg.de>
Date: Wed, 19 Apr 2023 09:30:02 +0200
Subject: [PATCH 33/35] Assert that time series sum matches cluster value

---
 etrago/cluster/disaggregation.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index a4144ae8..407e0316 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -741,6 +741,12 @@ def solve_partial_network(
                             for bus_id in filtered.index
                         }
                     )
+                    delta = abs((new_columns.sum(axis=1) - clt).sum())
+                    epsilon = 1e-5
+                    assert delta < epsilon, (
+                        "Sum of disaggregated time series does not match"
+                        f" aggregated timeseries: {delta=} > {epsilon=}."
+                    )
                     pn_t[s].loc[:, new_columns.columns] = new_columns
 
     def transfer_results(self, *args, **kwargs):

From b45c1d36e51f66879429b4ed06e18b0913f4cabb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Fri, 21 Apr 2023 11:40:40 +0200
Subject: [PATCH 34/35] Make string containing "{expression}" an f-string

It was obviously meant that way, so this is just a bugfix.
Add punctuation and a unit for better readability, too.
---
 etrago/cluster/disaggregation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index 407e0316..dfd635ac 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -347,7 +347,7 @@ def solve(self, scenario, solver):
                 )
         profile.disable()
         self.stats["check"] = time.time() - t
-        log.info("Checks computed in {self.stats['check']}")
+        log.info(f"Checks computed in {self.stats['check']}s.")
 
         profile.print_stats(sort="cumtime")
 

From 71e2f8c81f03758f4ce1f6fd4d6f4f0af9d84c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stephan=20G=C3=BCnther?= <gnn.code@gmail.com>
Date: Sun, 23 Apr 2023 22:54:35 +0200
Subject: [PATCH 35/35] Add links to "clustered - disaggregated" checks

This code prints the differences of the sum of all values of a certain
attribute or timeseries in the clustered network to the sum of the
corresponding disaggregated values in the original network.
Usually these should be close to zero. For `links`'s "p0" and "p1"
timeseries these aren't close to zero, but that's OK as these include
values from the CH4 links, which aren't disaggregated.
But it's probably still a good idea to have those printed in order to be
able to check whether these differences are in the correct ballpark.
Maybe the check can later be extended to be filtered so that they are
close to zero again.
---
 etrago/cluster/disaggregation.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/etrago/cluster/disaggregation.py b/etrago/cluster/disaggregation.py
index dfd635ac..054525fb 100644
--- a/etrago/cluster/disaggregation.py
+++ b/etrago/cluster/disaggregation.py
@@ -320,6 +320,7 @@ def solve(self, scenario, solver):
         for bt, ts in (
             ("generators", {"p": fs, "q": fs}),
             ("storage_units", {"p": fs, "state_of_charge": fs, "q": fs}),
+            ("links", {"p0": fs, "p1": fs}),
         ):
             log.info(f"Attribute sums, {bt}, clustered - disaggregated:")
             cnb = getattr(self.clustered_network, bt)