Merge branch 'dev' into features/#903-desaggregate-heat-pumps

CHANGELOG.rst

@@ -201,6 +201,10 @@ Added
   `#382 <https://github.com/openego/eGon-data/issues/382>`_
 * Add motorized individual travel
   `#553 <https://github.com/openego/eGon-data/issues/553>`_
+* Allocating MaStR PV rooftop power plants to OSM and synthetic buildings.
+  Desaggregating PV rooftop scenarios to mv grid districts and OSM and synthetic
+  buildings.
+  `#684 <https://github.com/openego/eGon-data/issues/684>`_
 * Add mapping zensus - weather cells
   `#845 <https://github.com/openego/eGon-data/issues/845>`_
 * Add pv rooftop plants per mv grid for eGon100RE
@@ -215,6 +219,8 @@ Added
   `#919 <https://github.com/openego/eGon-data/issues/919>`_
 * Add distribution of CHP plants for eGon100RE
   `#851 <https://github.com/openego/eGon-data/issues/851>`_
+* Add mapping table for all used buildings
+  `#962 <https://github.com/openego/eGon-data/issues/962>`_
 * Add charging infrastructure for e-mobility
   `#937 <https://github.com/openego/eGon-data/issues/937>`_
 * Add zipfile check
@@ -225,6 +231,14 @@ Added
   `#868 <https://github.com/openego/eGon-data/issues/868>`_
 * Write simBEV metadata to DB table
   `PR #978 <https://github.com/openego/eGon-data/pull/978>`_
+*  Add voltage level for electricity building loads
+  `#955 <https://github.com/openego/eGon-data/issues/955>`_
+* Add desaggregation of pv home batteries onto buildings
+  `#988 <https://github.com/openego/eGon-data/issues/988>`_
+* Add load areas
+  `#1014 <https://github.com/openego/eGon-data/issues/1014>`_
+* Add new MaStR dataset
+  `#1051 <https://github.com/openego/eGon-data/issues/1051>`_
 * Heat pump desaggregation to buildings
   `PR #903 <https://github.com/openego/eGon-data/pull/903>`_
 
@@ -430,10 +444,20 @@ Changed
 * Update creation of heat demand timeseries
   `#857 <https://github.com/openego/eGon-data/issues/857>`_
   `#856 <https://github.com/openego/eGon-data/issues/856>`_
+* Adjust H2 industry profiles abroad for eGon2035
+  `#940 <https://github.com/openego/eGon-data/issues/940>`_
 * Introduce carrier name 'others'
   `#819 <https://github.com/openego/eGon-data/issues/819>`_
 * Add rural heat pumps per medium voltage grid district
   `#987 <https://github.com/openego/eGon-data/issues/987>`_
+* Add eGon2021 scenario to demandregio dataset
+  `#1035 <https://github.com/openego/eGon-data/issues/1035>`_
+* Update MaStR dataset
+  `#519 <https://github.com/openego/eGon-data/issues/519>`_
+* Add missing VOM costs for heat sector components
+  `#942 <https://github.com/openego/eGon-data/issues/942>`_
+* Desaggregate industry demands to OSM areas and industrial sites
+  `#1001 <https://github.com/openego/eGon-data/issues/1001>`_
 
 
 Bug Fixes
@@ -603,7 +627,7 @@ Bug Fixes
 * Fix model load timeseries in motorized individual travel
   `#830 <https://github.com/openego/eGon-data/issues/830>`_
 * Fix gas costs
-  `#830 <https://github.com/openego/eGon-data/issues/847>`_
+  `#847 <https://github.com/openego/eGon-data/issues/847>`_
 * Add imports that have been wrongly deleted
   `#849 <https://github.com/openego/eGon-data/issues/849>`_
 * Fix final demand of heat demand timeseries
@@ -626,10 +650,25 @@ Bug Fixes
   `#914 <https://github.com/openego/eGon-data/issues/914>`_
 * Drop era5 weather cell table using cascade
   `#909 <https://github.com/openego/eGon-data/issues/909>`_
+* Remove drop of p_set and q_set for loads without timeserie
+  `#971 <https://github.com/openego/eGon-data/issues/971>`_
 * Delete gas bus with wrong country code
   `#958 <https://github.com/openego/eGon-data/issues/958>`_
 * Overwrite capacities for conventional power plants with data from nep list
   `#403 <https://github.com/openego/eGon-data/issues/403>`_
+* Adjust p_nom and marginal cost for OCGT in eGon2035
+  `#863 <https://github.com/openego/eGon-data/issues/863>`_
+* Mismatch of building bus_ids from cts_heat_demand_building_share
+  and mapping table
+  `#989 <https://github.com/openego/eGon-data/issues/989>`_
+* Fix zensus weather cells mapping
+  `#1031 <https://github.com/openego/eGon-data/issues/1031>`_
+* Fix solar rooftop in test mode
+  `#1055 <https://github.com/openego/eGon-data/issues/1055>`_
+* Add missing filter for scenario name in chp expansion
+  `#1015 <https://github.com/openego/eGon-data/issues/1015>`_
+* Fix installed capacity per individual heat pump
+  `#1058 <https://github.com/openego/eGon-data/issues/1058>`_
 
 .. _PR #692: https://github.com/openego/eGon-data/pull/692
 .. _#343: https://github.com/openego/eGon-data/issues/343

setup.py

@@ -5,7 +5,8 @@
 from glob import glob
 from os.path import basename, dirname, join, splitext
 import io
-import re 
+import re
+
 from setuptools import find_packages, setup
 
 
@@ -106,8 +107,9 @@ def read(*names, **kwargs):
         "rioxarray",
         "rtree",
         "saio",
+        "seaborn",
         "shapely",
-        "snakemake",
+        "snakemake<7",
         "sqlalchemy<1.4",  # Airflow<2.0 is not compatible with SQLAlchemy>=1.4
         "wtforms<3",  # WTForms>=3.0 breaks Airflow<2.0
         "xarray",

src/egon/data/airflow/dags/pipeline.py

@@ -68,7 +68,7 @@
 )
 from egon.data.datasets.industrial_sites import MergeIndustrialSites
 from egon.data.datasets.industry import IndustrialDemandCurves
-from egon.data.datasets.loadarea import LoadArea
+from egon.data.datasets.loadarea import LoadArea, OsmLanduse
 from egon.data.datasets.mastr import mastr_data_setup
 from egon.data.datasets.mv_grid_districts import mv_grid_districts_setup
 from egon.data.datasets.osm import OpenStreetMap
@@ -84,7 +84,7 @@
 from egon.data.datasets.scenario_capacities import ScenarioCapacities
 from egon.data.datasets.scenario_parameters import ScenarioParameters
 from egon.data.datasets.society_prognosis import SocietyPrognosis
-from egon.data.datasets.storages import PumpedHydro
+from egon.data.datasets.storages import Storages
 from egon.data.datasets.storages_etrago import StorageEtrago
 from egon.data.datasets.substation import SubstationExtraction
 from egon.data.datasets.substation_voronoi import SubstationVoronoi
@@ -215,7 +215,7 @@
     )
 
     # Extract landuse areas from the `osm` dataset
-    load_area = LoadArea(dependencies=[osm, vg250])
+    osm_landuse = OsmLanduse(dependencies=[osm, vg250])
 
     # Calculate feedin from renewables
     renewable_feedin = RenewableFeedin(
@@ -305,7 +305,7 @@
         dependencies=[
             demandregio,
             industrial_sites,
-            load_area,
+            osm_landuse,
             mv_grid_districts,
             osm,
         ]
@@ -475,7 +475,7 @@
             demand_curves_industry,
             district_heating_areas,
             industrial_sites,
-            load_area,
+            osm_landuse,
             mastr_data,
             mv_grid_districts,
             scenario_capacities,
@@ -531,7 +531,7 @@
     )
 
     # Pumped hydro units
-    pumped_hydro = PumpedHydro(
+    pumped_hydro = Storages(
         dependencies=[
             mastr_data,
             mv_grid_districts,
@@ -628,6 +628,40 @@
         ]
     )
 
+    mit_charging_infrastructure = MITChargingInfrastructure(
+        dependencies=[mv_grid_districts, hh_demand_buildings_setup]
+    )
+
+    # eMobility: heavy duty transport
+    heavy_duty_transport = HeavyDutyTransport(
+        dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035]
+    )
+
+    cts_demand_buildings = CtsDemandBuildings(
+        dependencies=[
+            osm_buildings_streets,
+            cts_electricity_demand_annual,
+            hh_demand_buildings_setup,
+            tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"],
+        ]
+    )
+
+    # Create load areas
+    load_areas = LoadArea(
+        dependencies=[
+            osm_landuse,
+            zensus_vg250,
+            household_electricity_demand_annual,
+            tasks[
+                "electricity_demand_timeseries"
+                ".hh_buildings"
+                ".get-building-peak-loads"
+            ],
+            cts_demand_buildings,
+            demand_curves_industry,
+        ]
+    )
+
     # ########## Keep this dataset at the end
     # Sanity Checks
     sanity_checks = SanityChecks(

src/egon/data/datasets.yml

@@ -297,6 +297,15 @@ mastr:
   file_basename: "bnetza_mastr"
   deposit_id: 808086
 
+mastr_new:
+  technologies:
+    - "wind"
+    - "hydro"
+    - "solar"
+    - "biomass"
+  file_basename: "bnetza_mastr"
+  deposit_id: 1132987
+
 re_potential_areas:
   target:
     schema: "supply"
@@ -346,6 +355,8 @@ hotmaps_current_policy_scenario_heat_demands_buildings:
 
 power_plants:
   sources:
+      mastr_pv: "bnetza_mastr_solar_cleaned.csv"
+      mastr_wind: "bnetza_mastr_wind_cleaned.csv"
       mastr_biomass: "bnetza_mastr_biomass_cleaned.csv"
       mastr_hydro: "bnetza_mastr_hydro_cleaned.csv"
       mastr_location: "location_elec_generation_raw.csv"
@@ -681,7 +692,7 @@ distributed_industrial_demand:
     demandregio:
       schema: 'demand'
       table: 'egon_demandregio_cts_ind'
-      scenarios: ["eGon2035", "eGon100RE"]
+      scenarios: ["eGon2021", "eGon2035", "eGon100RE"]
     wz:
       schema: 'demand'
       table: 'egon_demandregio_wz'
@@ -731,13 +742,22 @@ electrical_load_curves_industry:
     egon_mv_grid_district:
       schema: 'grid'
       table: 'egon_mv_grid_district'
+    egon_ehv_voronoi:
+      schema: 'grid'
+      table: 'egon_ehv_substation_voronoi'
   targets:
     osm_load:
       schema: 'demand'
       table: 'egon_osm_ind_load_curves'
+    osm_load_individual:
+      schema: 'demand'
+      table: 'egon_osm_ind_load_curves_individual'
     sites_load:
       schema: 'demand'
       table: 'egon_sites_ind_load_curves'
+    sites_load_individual:
+      schema: 'demand'
+      table: 'egon_sites_ind_load_curves_individual'
 
 etrago_electricity:
   sources:
@@ -1157,7 +1177,9 @@ mobility_hgv:
       # share of FCEV trucks
       # within the egon project the flat guess is 100%
       fcev_share: 1.
-      scenarios: ["eGon2035", "eGon100RE"]
+      scenarios:
+        - "eGon2035"
+        - "eGon100RE"
       carrier: "H2_hgv_load"
       energy_value_h2: 39.4  # kWh/kg
       hours_per_year: 8760
@@ -1174,3 +1196,24 @@ mobility_hgv:
       # Langfristszenarien
       # https://www.langfristszenarien.de/enertile-explorer-wAssets/docs/LFS3_Langbericht_Verkehr_final.pdf#page=17
       eGon100RE: 40000000000
+
+home_batteries:
+  constants:
+    scenarios:
+      - "eGon2035"
+      - "eGon100RE"
+    # Mean ratio between the storage capacity and the power of the pv rooftop system
+    cbat_ppv_ratio: 1
+    rtol: 0.05
+    max_it: 100
+  sources:
+    etrago_storage:
+      schema: 'grid'
+      table: 'egon_etrago_storage'
+    storage:
+      schema: 'supply'
+      table: 'egon_storages'
+  targets:
+    home_batteries:
+      schema: 'supply'
+      table: 'egon_home_batteries'

src/egon/data/datasets/DSM_cts_ind.py

@@ -212,7 +212,7 @@ def calc_ind_site_timeseries(scenario):
                 data=curves_bus["bus_id"], index=curves_bus["id"].astype(int)
             )
             ts["scenario_name"] = scenario
-            curves_bus.drop({"id", "bus_id"}, axis=1, inplace=True)
+            curves_bus.drop({"id", "bus_id", "geom"}, axis=1, inplace=True)
             ts["p_set"] = curves_bus.values.tolist()
 
             # add subsector to relate to Schmidt's tables afterwards

src/egon/data/datasets/chp/__init__.py

@@ -3,6 +3,8 @@
 (CHP) plants.
 """
 
+from pathlib import Path
+
 from geoalchemy2 import Geometry
 from shapely.ops import nearest_points
 from sqlalchemy import Boolean, Column, Float, Integer, Sequence, String
@@ -11,6 +13,7 @@
 from sqlalchemy.orm import sessionmaker
 import geopandas as gpd
 import pandas as pd
+import pypsa
 
 from egon.data import config, db
 from egon.data.datasets import Dataset
@@ -19,17 +22,16 @@
     assign_use_case,
     existing_chp_smaller_10mw,
     extension_per_federal_state,
+    extension_to_areas,
     select_target,
 )
+from egon.data.datasets.mastr import WORKING_DIR_MASTR_OLD
 from egon.data.datasets.power_plants import (
     assign_bus_id,
     assign_voltage_level,
     filter_mastr_geometry,
     scale_prox2now,
 )
-import pypsa
-from egon.data.datasets.chp.small_chp import extension_to_areas
-from pathlib import Path
 
 Base = declarative_base()
 
@@ -248,9 +250,9 @@ def insert_biomass_chp(scenario):
     target = select_target("biomass", scenario)
 
     # import data for MaStR
-    mastr = pd.read_csv(cfg["sources"]["mastr_biomass"]).query(
-        "EinheitBetriebsstatus=='InBetrieb'"
-    )
+    mastr = pd.read_csv(
+        WORKING_DIR_MASTR_OLD / cfg["sources"]["mastr_biomass"]
+    ).query("EinheitBetriebsstatus=='InBetrieb'")
 
     # Drop entries without federal state or 'AusschließlichWirtschaftszone'
     mastr = mastr[
@@ -278,7 +280,9 @@ def insert_biomass_chp(scenario):
 
     # Assign bus_id
     if len(mastr_loc) > 0:
-        mastr_loc["voltage_level"] = assign_voltage_level(mastr_loc, cfg)
+        mastr_loc["voltage_level"] = assign_voltage_level(
+            mastr_loc, cfg, WORKING_DIR_MASTR_OLD
+        )
         mastr_loc = assign_bus_id(mastr_loc, cfg)
     mastr_loc = assign_use_case(mastr_loc, cfg["sources"])
 
@@ -528,7 +532,7 @@ class Chp(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="Chp",
-            version="0.0.5",
+            version="0.0.6",
             dependencies=dependencies,
             tasks=(
                 create_tables,