Merge fixes/#589 into CI branch

src/egon/data/airflow/dags/pipeline.py

@@ -157,21 +157,24 @@
 
     saltcavern_storage = SaltcavernData(dependencies=[data_bundle, vg250])
 
-    # NEP data import
-    scenario_capacities = ScenarioCapacities(
-        dependencies=[setup, vg250, data_bundle, zensus_population]
+    # Import weather data
+    weather_data = WeatherData(
+        dependencies=[setup, scenario_parameters, vg250]
     )
+    download_weather_data = tasks["era5.download-era5"]
+
+    # Future national heat demands for foreign countries based on Hotmaps
+    # download only, processing in PyPSA-Eur-Sec fork
+    hd_abroad = HeatDemandEurope(dependencies=[setup])
+
+    # Download industrial gas demand
+    industrial_gas_demand = IndustrialGasDemand(dependencies=[setup])
 
     # setting etrago input tables
 
     setup_etrago = EtragoSetup(dependencies=[setup])
     etrago_input_data = tasks["etrago_setup.create-tables"]
 
-    # Retrieve MaStR data
-    mastr_data = mastr_data_setup(dependencies=[setup])
-    mastr_data.insert_into(pipeline)
-    retrieve_mastr_data = tasks["mastr.download-mastr-data"]
-
     substation_extraction = SubstationExtraction(
         dependencies=[osm_add_metadata, vg250_clean_and_prepare]
     )
@@ -189,6 +192,33 @@
     osmtgmod_pypsa = tasks["osmtgmod.to-pypsa"]
     osmtgmod_substation = tasks["osmtgmod_substation"]
 
+    # run pypsa-eur-sec
+    run_pypsaeursec = PypsaEurSec(
+        dependencies=[
+            weather_data,
+            hd_abroad,
+            osmtgmod,
+            setup_etrago,
+            data_bundle,
+        ]
+    )
+
+    # NEP data import
+    scenario_capacities = ScenarioCapacities(
+        dependencies=[
+            setup,
+            vg250,
+            data_bundle,
+            zensus_population,
+            run_pypsaeursec,
+        ]
+    )
+
+    # Retrieve MaStR data
+    mastr_data = mastr_data_setup(dependencies=[setup])
+    mastr_data.insert_into(pipeline)
+    retrieve_mastr_data = tasks["mastr.download-mastr-data"]
+
     # create Voronoi polygons
     substation_voronoi = SubstationVoronoi(
         dependencies=[osmtgmod_substation, vg250]
@@ -214,24 +244,9 @@
         dependencies=[vg250, scenario_parameters, zensus_vg250]
     )
 
-    # Future national heat demands for foreign countries based on Hotmaps
-    # download only, processing in PyPSA-Eur-Sec fork
-    hd_abroad = HeatDemandEurope(dependencies=[setup])
-    hd_abroad.insert_into(pipeline)
-    heat_demands_abroad_download = tasks["heat_demand_europe.download"]
-
-    # Download industrial gas demand
-    industrial_gas_demand = IndustrialGasDemand(dependencies=[setup])
-
     # Extract landuse areas from osm data set
     load_area = LoadArea(dependencies=[osm, vg250])
 
-    # Import weather data
-    weather_data = WeatherData(
-        dependencies=[setup, scenario_parameters, vg250]
-    )
-    download_weather_data = tasks["era5.download-era5"]
-
     renewable_feedin = RenewableFeedin(dependencies=[weather_data, vg250])
 
     feedin_wind_onshore = tasks["renewable_feedin.wind"]
@@ -269,7 +284,7 @@
         dependencies=[vg250, mv_grid_districts]
     )
 
-    #
+
     mv_hh_electricity_load_2035 = PythonOperator(
         task_id="MV-hh-electricity-load-2035",
         python_callable=hh_profiles.mv_grid_district_HH_electricity_load,
@@ -307,7 +322,7 @@
 
     # Household electricity demand buildings
     hh_demand_buildings_setup = hh_buildings.setup(
-        dependencies=[householdprofiles_in_cencus_cells],
+        dependencies=[householdprofiles_in_cencus_cells]
     )
 
     hh_demand_buildings_setup.insert_into(pipeline)
@@ -339,6 +354,21 @@
     elec_cts_demands_zensus = tasks[
         "electricity_demand.distribute-cts-demands"
     ]
+
+    mv_hh_electricity_load_2035 = PythonOperator(
+        task_id="MV-hh-electricity-load-2035",
+        python_callable=hh_profiles.mv_grid_district_HH_electricity_load,
+        op_args=["eGon2035", 2035],
+        op_kwargs={"drop_table": True},
+    )
+
+    mv_hh_electricity_load_2050 = PythonOperator(
+        task_id="MV-hh-electricity-load-2050",
+        python_callable=hh_profiles.mv_grid_district_HH_electricity_load,
+        op_args=["eGon100RE", 2050],
+    )
+
+
     # Industry
 
     industrial_sites = MergeIndustrialSites(
@@ -356,6 +386,7 @@
     )
 
     # Electrical loads to eTraGo
+
     electrical_load_etrago = ElectricalLoadEtrago(
         dependencies=[
             demand_curves_industry,
@@ -364,17 +395,6 @@
         ]
     )
 
-    # run pypsa-eur-sec
-    run_pypsaeursec = PypsaEurSec(
-        dependencies=[
-            weather_data,
-            hd_abroad,
-            osmtgmod,
-            setup_etrago,
-            data_bundle,
-        ]
-    )
-
     foreign_lines = ElectricalNeighbours(
         dependencies=[run_pypsaeursec, tyndp_data]
     )
@@ -411,16 +431,12 @@
 
     # Power-to-gas-to-power chain installations
     insert_power_to_h2_installations = HydrogenPowerLinkEtrago(
-        dependencies=[
-            insert_hydrogen_buses,
-        ]
+        dependencies=[insert_hydrogen_buses]
     )
 
     # Link between methane grid and respective hydrogen buses
     insert_h2_to_ch4_grid_links = HydrogenMethaneLinkEtrago(
-        dependencies=[
-            insert_hydrogen_buses,
-        ]
+        dependencies=[insert_hydrogen_buses]
     )
 
     insert_h2_grid = HydrogenGridEtrago(
@@ -597,9 +613,5 @@
     # Storages to eTrago
 
     storage_etrago = StorageEtrago(
-        dependencies=[
-            pumped_hydro,
-            setup_etrago,
-            scenario_parameters,
-        ]
+        dependencies=[pumped_hydro, setup_etrago, scenario_parameters]
     )

src/egon/data/datasets.yml

@@ -260,13 +260,25 @@ society_prognosis:
       table: 'egon_household_prognosis'
 
 scenario_input:
-  eGon2035:
-    paths:
-      "capacities": "NEP2035_V2021_scnC2035.xlsx"
-      "list_conv_pp": "Kraftwerksliste_NEP_2021_konv.csv"
-  eGon100RE:
-    paths:
-      "capacities": "nodal_capacities.csv"
+  sources:
+    eGon2035:
+      capacities: "NEP2035_V2021_scnC2035.xlsx"
+      list_conv_pp: "Kraftwerksliste_NEP_2021_konv.csv"
+    eGon100RE:
+      capacities: "nodal_capacities.csv"
+    boundaries:
+      schema: 'boundaries'
+      table: 'vg250_lan'
+    zensus_population:
+      schema: 'society'
+      table: 'destatis_zensus_population_per_ha'
+  targets:
+    scenario_capacities:
+      schema: 'supply'
+      table: 'egon_scenario_capacities'
+    nep_conventional_powerplants:
+      schema: 'supply'
+      table: 'egon_nep_2021_conventional_powerplants'
 
 mastr:
   technologies: