Merge pull request #225 from openego/features/#177-create-scenario-table

Integrate scenario table

CHANGELOG.rst

@@ -70,12 +70,12 @@ Added
   `#181 <https://github.com/openego/eGon-data/issues/181>`_
 * Distribute electrical demands of cts to zensus cells
   `#210 <https://github.com/openego/eGon-data/issues/210>`_
-
-.. _PR #159: https://github.com/openego/eGon-data/pull/159
-
 * Include industrial sites' download, import and merge
   `#117 <https://github.com/openego/eGon-data/issues/117>`_
+* Integrate scenario table with parameters for each sector 
+  `#177 <https://github.com/openego/eGon-data/issues/177>`_
 
+.. _PR #159: https://github.com/openego/eGon-data/pull/159
 
 Changed
 -------

src/egon/data/airflow/dags/pipeline.py

@@ -30,6 +30,7 @@
 import egon.data.processing.zensus_vg250.zensus_population_inside_germany as zensus_vg250
 import egon.data.importing.re_potential_areas as re_potential_areas
 import egon.data.importing.heat_demand_data as import_hd
+import egon.data.importing.scenarios as import_scenarios
 
 
 import egon.data.importing.industrial_sites as industrial_sites
@@ -161,34 +162,49 @@
     ] >> map_zensus_vg250 >> zensus_inside_ger >> zensus_inside_ger_metadata
     zensus_inside_ger >> vg250_population >> vg250_population_metadata
 
+    # Scenario table
+    scenario_input_tables = PythonOperator(
+        task_id="create-scenario-parameters-table",
+        python_callable=import_scenarios.create_table
+    )
+
+    scenario_input_import = PythonOperator(
+        task_id="import-scenario-parameters",
+        python_callable=import_scenarios.insert_scenarios
+    )
+    setup >> scenario_input_tables >> scenario_input_import
+
     # DemandRegio data import
     demandregio_tables = PythonOperator(
         task_id="demandregio-tables",
         python_callable=import_dr.create_tables,
     )
 
-    setup >> demandregio_tables
+    scenario_input_tables >> demandregio_tables
 
     demandregio_society = PythonOperator(
         task_id="demandregio-society",
         python_callable=import_dr.insert_society_data,
     )
     vg250_clean_and_prepare >> demandregio_society
     demandregio_tables >> demandregio_society
+    scenario_input_import >> demandregio_society
 
     demandregio_demand_households = PythonOperator(
         task_id="demandregio-household-demands",
         python_callable=import_dr.insert_household_demand,
     )
     vg250_clean_and_prepare >> demandregio_demand_households
     demandregio_tables >> demandregio_demand_households
+    scenario_input_import >> demandregio_demand_households
 
     demandregio_demand_cts_ind = PythonOperator(
         task_id="demandregio-cts-industry-demands",
         python_callable=import_dr.insert_cts_ind_demands,
     )
     vg250_clean_and_prepare >> demandregio_demand_cts_ind
     demandregio_tables >> demandregio_demand_cts_ind
+    scenario_input_import >> demandregio_demand_cts_ind
 
     # Society prognosis
     prognosis_tables = PythonOperator(
@@ -341,6 +357,7 @@
     )
     vg250_clean_and_prepare >> heat_demand_import
     zensus_inside_ger_metadata >> heat_demand_import
+    scenario_input_import >> heat_demand_import
 
     # Power plant setup
     power_plant_tables = PythonOperator(
@@ -375,7 +392,7 @@
     vg250_clean_and_prepare >> industrial_sites_import
     industrial_sites_import >> industrial_sites_merge >> industrial_sites_nuts
 
-# Districute electrical CTS demands to zensus grid
+    # Distribute electrical CTS demands to zensus grid
 
     elec_cts_demands_zensus = PythonOperator(
         task_id="electrical-cts-demands-zensus",

src/egon/data/datasets.yml

@@ -76,15 +76,17 @@ demandregio_society:
     vg250_krs:
       schema: 'boundaries'
       table: 'vg250_krs'
+    scenarios:
+      schema: 'scenario'
+      table: 'egon_scenario_parameters'
   targets:
     population:
       schema: 'society'
       table: 'egon_demandregio_population'
-      target_years: [2018, 2035, 2050]
     household:
       schema: 'society'
       table: 'egon_demandregio_household'
-      target_years: [2018, 2035, 2050]
+
 
 demandregio_household_demand:
   sources:
@@ -93,11 +95,13 @@ demandregio_household_demand:
     vg250_krs:
       schema: 'boundaries'
       table: 'vg250_krs'
+    scenarios:
+      schema: 'scenario'
+      table: 'egon_scenario_parameters'
   targets:
     household_demand:
       schema: 'demand'
       table: 'egon_demandregio_hh'
-      scenarios: ["eGon2035", "eGon100RE"]
 
 demandregio_cts_ind_demand:
   sources:
@@ -110,11 +114,13 @@ demandregio_cts_ind_demand:
     wz_definitions:
       "CTS": 'CTS_WZ_definition.csv'
       "industry": 'ind_WZ_definition.csv'
+    scenarios:
+      schema: 'scenario'
+      table: 'egon_scenario_parameters'
   targets:
     cts_ind_demand:
       schema: 'demand'
       table: 'egon_demandregio_cts_ind'
-      scenarios: ["eGon2035", "eGon100RE"]
     wz_definitions:
       schema: 'demand'
       table: 'egon_demandregio_wz'

src/egon/data/importing/demandregio/__init__.py

@@ -4,11 +4,14 @@
 """
 import os
 import pandas as pd
+import numpy as np
 import egon.data.config
 from egon.data import db
-from sqlalchemy import Column, String, Float, Integer
+from egon.data.importing.scenarios import get_sector_parameters, EgonScenario
+from sqlalchemy import Column, String, Float, Integer, ForeignKey
 from sqlalchemy.ext.declarative import declarative_base
 from disaggregator import data, spatial
+import egon.data.importing.scenarios.parameters as scenario_parameters
 # will be later imported from another file ###
 Base = declarative_base()
 
@@ -18,7 +21,7 @@ class EgonDemandRegioHH(Base):
     __table_args__ = {'schema': 'demand'}
     nuts3 = Column(String(5), primary_key=True)
     hh_size = Column(Integer, primary_key=True)
-    scenario = Column(String(50), primary_key=True)
+    scenario = Column(String, ForeignKey(EgonScenario.name), primary_key=True)
     year = Column(Integer)
     demand = Column(Float)
 
@@ -28,7 +31,7 @@ class EgonDemandRegioCtsInd(Base):
     __table_args__ = {'schema': 'demand'}
     nuts3 = Column(String(5), primary_key=True)
     wz = Column(Integer, primary_key=True)
-    scenario = Column(String(50), primary_key=True)
+    scenario = Column(String, ForeignKey(EgonScenario.name), primary_key=True)
     year = Column(Integer)
     demand = Column(Float)
 
@@ -385,14 +388,9 @@ def insert_household_demand():
         db.execute_sql(
                 f"DELETE FROM {targets[t]['schema']}.{targets[t]['table']};")
 
-    for scn in targets['household_demand']['scenarios']:
+    for scn in ['eGon2035', 'eGon100RE']:
 
-        if scn == 'eGon2035':
-            year = 2035
-        elif scn == 'eGon100RE':
-            year = 2050
-        else:
-            print(f"Warning: Scenario {scn} can not be imported.")
+        year = scenario_parameters.global_settings(scn)['population_year']
 
         # Insert demands of private households
         insert_hh_demand(scn, year, engine)
@@ -417,14 +415,12 @@ def insert_cts_ind_demands():
 
     insert_cts_ind_wz_definitions()
 
-    for scn in targets['cts_ind_demand']['scenarios']:
+    for scn in ['eGon2035', 'eGon100RE']:
 
-        if scn == 'eGon2035':
-            year = 2035
-        elif scn == 'eGon100RE':
+        year = scenario_parameters.global_settings(scn)['population_year']
+
+        if year > 2035:
             year = 2035
-        else:
-            print(f"Warning: Scenario {scn} can not be imported.")
 
         # target values per scenario in MWh
         target_values = {
@@ -458,8 +454,10 @@ def insert_society_data():
         db.execute_sql(
                 f"DELETE FROM {targets[t]['schema']}.{targets[t]['table']};")
 
+    target_years = np.append(
+        get_sector_parameters('global').population_year.values, 2018)
 
-    for year in targets['population']['target_years']:
+    for year in target_years:
         df_pop = pd.DataFrame(data.population(year=year))
         df_pop['year'] = year
         df_pop = df_pop.rename({'value': 'population'}, axis='columns')
@@ -471,7 +469,7 @@ def insert_society_data():
                       if_exists='append')
 
 
-    for year in targets['household']['target_years']:
+    for year in target_years:
         df_hh = pd.DataFrame(data.households_per_size(year=year))
         # Select data for nuts3-regions in boundaries (needed for testmode)
         df_hh = data_in_boundaries(df_hh)

src/egon/data/importing/heat_demand_data/__init__.py

@@ -16,11 +16,11 @@
 
 from egon.data import db, subprocess
 import egon.data.config
+from egon.data.importing.scenarios import get_sector_parameters, EgonScenario
 from urllib.request import urlretrieve
 import os
 import zipfile
 
-import pandas as pd
 import geopandas as gpd
 
 # for raster operations
@@ -34,7 +34,7 @@
 # import time
 
 # packages for ORM class definition
-from sqlalchemy import Column, String, Float, Integer, Sequence
+from sqlalchemy import Column, String, Float, Integer, Sequence, ForeignKey
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
@@ -53,7 +53,7 @@ class EgonPetaHeat(Base):
     )
     demand = Column(Float)
     sector = Column(String)
-    scenario = Column(String)
+    scenario = Column(String, ForeignKey(EgonScenario.name))
     version = Column(String)
     zensus_population_id = Column(Integer)
 
@@ -308,7 +308,7 @@ def future_heat_demand_germany(scenario_name):
 
     The calculation is based on Peta5_0_1 heat demand densities, cutcut for
     Germany, for the year 2015. The given scenario name is used to read the
-    adjustment factors for the heat demand rasters from a file.
+    adjustment factors for the heat demand rasters from the scenario table.
 
     Parameters
     ----------
@@ -333,34 +333,14 @@ def future_heat_demand_germany(scenario_name):
 
         Check, if there are populated cells without heat demand.
 
-        Check, if there are unpoplated cells with residential heat demands.
-
-        Option: Load the adjustment factors from an excel files, and not from
-        a csv: That might be implemented later for better documentation of the
-        development of the adjustment factors.
-        pip install xlrd to make it work
-        xlsfilename = ""
-        df_reductions = pd.read_excel(xlsfilename,
-                                      sheet_name =
-                                      "scenarios_for_raster_adjustment")
     """
+    # Load the values
+    heat_parameters = get_sector_parameters('heat', scenario=scenario_name)
+
+    res_hd_reduction = heat_parameters['DE_demand_reduction_residential']
+
+    ser_hd_reduction = heat_parameters['DE_demand_reduction_service']
 
-    # Load the csv file with the sceanario data for raster adjustment
-    csvfilename = os.path.join(
-        os.path.dirname(__file__), "scenarios_HD_raster_adjustments.csv"
-    )
-    df_reductions = pd.read_csv(csvfilename).set_index("scenario")
-
-    # Load the values, if scenario name is found in the file
-    if scenario_name in df_reductions.index:
-        res_hd_reduction = df_reductions.loc[
-            scenario_name, "HD_reduction_residential"
-        ]
-        ser_hd_reduction = df_reductions.loc[
-            scenario_name, "HD_reduction_service_sector"
-        ]
-    else:
-        print(f"Scenario {scenario_name} not defined.")
 
     # Define the directory where the created rasters will be saved
     scenario_raster_directory = "heat_scenario_raster"
@@ -781,6 +761,14 @@ def add_metadata():
                                 "resource": "society.destatis_zensus_population_per_ha",
                                 "fields": ["id"],
                             },
+                        },
+
+                        {
+                            "fields": ["scenario"],
+                            "reference": {
+                                "resource": "scenario.egon_scenario_parameters",
+                                "fields": ["name"],
+                            },
                         }
                     ],
                 },