Merge branch 'dev' into features/#929-H2-carrier-name

.pre-commit-config.yaml

@@ -5,21 +5,21 @@
 exclude: '^(\.tox|ci/templates|\.bumpversion\.cfg)(/|$)'
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
       - id: isort
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
-    hooks:
-      - id: flake8
   - repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 22.12.0
     hooks:
       - id: black
-        language_version: python3 # Should be a command that runs python3.6+
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        language_version: python3.8

setup.cfg

@@ -25,7 +25,7 @@ python_files =
     tests.py
 addopts =
     -ra
-    --strict
+    --strict-markers
     --ignore=docs/conf.py
     --ignore=setup.py
     --ignore=ci

setup.py

@@ -5,7 +5,8 @@
 from glob import glob
 from os.path import basename, dirname, join, splitext
 import io
-import re 
+import re
+
 from setuptools import find_packages, setup
 
 
@@ -78,7 +79,7 @@ def read(*names, **kwargs):
     keywords=[
         # eg: 'keyword1', 'keyword2', 'keyword3',
     ],
-    python_requires=">=3.7.*",
+    python_requires=">=3.7",
     install_requires=[
         # eg: 'aspectlib==1.1.1', 'six>=1.7',
         "apache-airflow>=1.10.14,<2.0",  # See accompanying commit message
@@ -106,8 +107,9 @@ def read(*names, **kwargs):
         "rioxarray",
         "rtree",
         "saio",
+        "seaborn",
         "shapely",
-        "snakemake",
+        "snakemake<7",
         "sqlalchemy<1.4",  # Airflow<2.0 is not compatible with SQLAlchemy>=1.4
         "wtforms<3",  # WTForms>=3.0 breaks Airflow<2.0
         "xarray",

src/egon/data/airflow/dags/pipeline.py

@@ -13,7 +13,7 @@
 from egon.data.datasets.data_bundle import DataBundle
 from egon.data.datasets.demandregio import DemandRegio
 from egon.data.datasets.district_heating_areas import DistrictHeatingAreas
-from egon.data.datasets.DSM_cts_ind import dsm_Potential
+from egon.data.datasets.DSM_cts_ind import DsmPotential
 from egon.data.datasets.electrical_neighbours import ElectricalNeighbours
 from egon.data.datasets.electricity_demand import (
     CtsElectricityDemand,
@@ -49,6 +49,11 @@
 from egon.data.datasets.heat_etrago import HeatEtrago
 from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable
 from egon.data.datasets.heat_supply import HeatSupply
+from egon.data.datasets.heat_supply.individual_heating import (
+    HeatPumps2035,
+    HeatPumps2050,
+    HeatPumpsPypsaEurSec,
+)
 from egon.data.datasets.hydrogen_etrago import (
     HydrogenBusEtrago,
     HydrogenGridEtrago,
@@ -63,7 +68,8 @@
 )
 from egon.data.datasets.industrial_sites import MergeIndustrialSites
 from egon.data.datasets.industry import IndustrialDemandCurves
-from egon.data.datasets.loadarea import LoadArea
+from egon.data.datasets.loadarea import LoadArea, OsmLanduse
+from egon.data.datasets.low_flex_scenario import LowFlexScenario
 from egon.data.datasets.mastr import mastr_data_setup
 from egon.data.datasets.mv_grid_districts import mv_grid_districts_setup
 from egon.data.datasets.osm import OpenStreetMap
@@ -79,7 +85,7 @@
 from egon.data.datasets.scenario_capacities import ScenarioCapacities
 from egon.data.datasets.scenario_parameters import ScenarioParameters
 from egon.data.datasets.society_prognosis import SocietyPrognosis
-from egon.data.datasets.storages import PumpedHydro
+from egon.data.datasets.storages import Storages
 from egon.data.datasets.storages_etrago import StorageEtrago
 from egon.data.datasets.substation import SubstationExtraction
 from egon.data.datasets.substation_voronoi import SubstationVoronoi
@@ -210,7 +216,7 @@
     )
 
     # Extract landuse areas from the `osm` dataset
-    load_area = LoadArea(dependencies=[osm, vg250])
+    osm_landuse = OsmLanduse(dependencies=[osm, vg250])
 
     # Calculate feedin from renewables
     renewable_feedin = RenewableFeedin(
@@ -300,7 +306,7 @@
         dependencies=[
             demandregio,
             industrial_sites,
-            load_area,
+            osm_landuse,
             mv_grid_districts,
             osm,
         ]
@@ -329,6 +335,24 @@
         ]
     )
 
+    cts_demand_buildings = CtsDemandBuildings(
+        dependencies=[
+            osm_buildings_streets,
+            cts_electricity_demand_annual,
+            hh_demand_buildings_setup,
+            tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"],
+        ]
+    )
+
+    # Minimum heat pump capacity for pypsa-eur-sec
+    heat_pumps_pypsa_eur_sec = HeatPumpsPypsaEurSec(
+        dependencies=[
+            cts_demand_buildings,
+            DistrictHeatingAreas,
+            heat_time_series,
+        ]
+    )
+
     # run pypsa-eur-sec
     run_pypsaeursec = PypsaEurSec(
         dependencies=[
@@ -339,6 +363,7 @@
             data_bundle,
             electrical_load_etrago,
             heat_time_series,
+            heat_pumps_pypsa_eur_sec,
         ]
     )
 
@@ -389,6 +414,7 @@
             create_gas_polygons_egon2035,
             gas_grid_insert_data,
             insert_hydrogen_buses,
+            run_pypsaeursec,
         ]
     )
 
@@ -419,7 +445,7 @@
             run_pypsaeursec,
             foreign_lines,
             insert_hydrogen_buses,
-            create_gas_polygons_egon100RE
+            create_gas_polygons_egon100RE,
         ]
     )
 
@@ -451,7 +477,7 @@
             demand_curves_industry,
             district_heating_areas,
             industrial_sites,
-            load_area,
+            osm_landuse,
             mastr_data,
             mv_grid_districts,
             scenario_capacities,
@@ -498,7 +524,7 @@
     )
 
     # DSM (demand site management)
-    components_dsm = dsm_Potential(
+    components_dsm = DsmPotential(
         dependencies=[
             cts_electricity_demand_annual,
             demand_curves_industry,
@@ -507,7 +533,7 @@
     )
 
     # Pumped hydro units
-    pumped_hydro = PumpedHydro(
+    pumped_hydro = Storages(
         dependencies=[
             mastr_data,
             mv_grid_districts,
@@ -533,19 +559,50 @@
     # CHP to eTraGo
     chp_etrago = ChpEtrago(dependencies=[chp, heat_etrago])
 
+    # Storages to eTraGo
+    storage_etrago = StorageEtrago(
+        dependencies=[pumped_hydro, scenario_parameters, setup_etrago]
+    )
+
+    mit_charging_infrastructure = MITChargingInfrastructure(
+        dependencies=[mv_grid_districts, hh_demand_buildings_setup]
+    )
+
+    # eMobility: heavy duty transport
+    heavy_duty_transport = HeavyDutyTransport(
+        dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035]
+    )
+
+    # Heat pump disaggregation for eGon2035
+    heat_pumps_2035 = HeatPumps2035(
+        dependencies=[
+            cts_demand_buildings,
+            DistrictHeatingAreas,
+            heat_supply,
+            heat_time_series,
+            heat_pumps_pypsa_eur_sec,
+            tasks["power_plants.pv_rooftop_buildings.pv-rooftop-to-buildings"],
+        ]
+    )
+
     # HTS to eTraGo table
     hts_etrago_table = HtsEtragoTable(
         dependencies=[
             district_heating_areas,
             heat_etrago,
             heat_time_series,
             mv_grid_districts,
+            heat_pumps_2035,
         ]
     )
 
-    # Storages to eTraGo
-    storage_etrago = StorageEtrago(
-        dependencies=[pumped_hydro, scenario_parameters, setup_etrago]
+    # Heat pump disaggregation for eGon100RE
+    heat_pumps_2050 = HeatPumps2050(
+        dependencies=[
+            run_pypsaeursec,
+            heat_pumps_pypsa_eur_sec,
+            heat_supply,
+        ]
     )
 
     # eMobility: motorized individual travel
@@ -591,6 +648,34 @@
         ]
     )
 
+    # Create load areas
+    load_areas = LoadArea(
+        dependencies=[
+            osm_landuse,
+            zensus_vg250,
+            household_electricity_demand_annual,
+            tasks[
+                "electricity_demand_timeseries"
+                ".hh_buildings"
+                ".get-building-peak-loads"
+            ],
+            cts_demand_buildings,
+            demand_curves_industry,
+        ]
+    )
+
+    # Include low flex scenario(s)
+    low_flex_scenario = LowFlexScenario(
+        dependencies=[
+            storage_etrago,
+            hts_etrago_table,
+            fill_etrago_generators,
+            household_electricity_demand_annual,
+            cts_demand_buildings,
+            emobility_mit,
+        ]
+    )
+
     # ########## Keep this dataset at the end
     # Sanity Checks
     sanity_checks = SanityChecks(
@@ -601,5 +686,6 @@
             household_electricity_demand_annual,
             cts_demand_buildings,
             emobility_mit,
+            low_flex_scenario,
         ]
     )