Merge branch 'dev' into features/#177-create-scenario-table

CHANGELOG.rst

@@ -68,6 +68,8 @@ Added
   `PR #164 <https://github.com/openego/eGon-data/pull/164>`_  
 * Distribute electrical demands of households to zensus cells
   `#181 <https://github.com/openego/eGon-data/issues/181>`_
+* Distribute electrical demands of cts to zensus cells
+  `#210 <https://github.com/openego/eGon-data/issues/210>`_
 
 .. _PR #159: https://github.com/openego/eGon-data/pull/159
 
@@ -109,3 +111,7 @@ Bug fixes
   `#171 <https://github.com/openego/eGon-data/issues/171>`_
 * Adjust names of demandregios nuts3 regions according to nuts version 2016
   `#201 <https://github.com/openego/eGon-data/issues/201>`_
+* Delete zensus buildings, apartments and households in unpopulated cells
+  `#202 <https://github.com/openego/eGon-data/issues/202>`_
+* Fix input table of electrical-demands-zensus
+  `#217 <https://github.com/openego/eGon-data/issues/217>`_

src/egon/data/airflow/dags/pipeline.py

@@ -3,32 +3,29 @@
 from airflow.operators.postgres_operator import PostgresOperator
 from airflow.operators.python_operator import PythonOperator
 from airflow.utils.dates import days_ago
-import airflow
 import importlib_resources as resources
 
 from egon.data.airflow.tasks import initdb
+from egon.data.processing.zensus_vg250 import (
+    zensus_population_inside_germany as zensus_vg250,
+)
+import airflow
 import egon.data.importing.demandregio as import_dr
 import egon.data.importing.etrago as etrago
+import egon.data.importing.heat_demand_data as import_hd
 import egon.data.importing.mastr as mastr
 import egon.data.importing.nep_input_data as nep_input
 import egon.data.importing.openstreetmap as import_osm
+import egon.data.importing.re_potential_areas as re_potential_areas
 import egon.data.importing.vg250 as import_vg250
-import egon.data.processing.openstreetmap as process_osm
 import egon.data.importing.zensus as import_zs
-import egon.data.processing.zensus as process_zs
+import egon.data.processing.demandregio as process_dr
+import egon.data.processing.openstreetmap as process_osm
+import egon.data.processing.osmtgmod as osmtgmod
 import egon.data.processing.power_plants as power_plants
-import egon.data.importing.nep_input_data as nep_input
-import egon.data.importing.etrago as etrago
-import egon.data.importing.mastr as mastr
 import egon.data.processing.substation as substation
-import egon.data.processing.zensus_vg250.zensus_population_inside_germany as zensus_vg250
-import egon.data.importing.re_potential_areas as re_potential_areas
-import egon.data.importing.heat_demand_data as import_hd
-import egon.data.processing.osmtgmod as osmtgmod
-import egon.data.processing.demandregio as process_dr
+import egon.data.processing.zensus as process_zs
 import egon.data.importing.scenarios as import_scenarios
-from egon.data import db
-
 
 with airflow.DAG(
     "egon-data-processing-pipeline",
@@ -202,14 +199,14 @@
     # Society prognosis
     prognosis_tables = PythonOperator(
         task_id="create-prognosis-tables",
-        python_callable=process_zs.create_tables
+        python_callable=process_zs.create_tables,
     )
 
     setup >> prognosis_tables
 
     population_prognosis = PythonOperator(
         task_id="zensus-population-prognosis",
-        python_callable=process_zs.population_prognosis_to_zensus
+        python_callable=process_zs.population_prognosis_to_zensus,
     )
 
     prognosis_tables >> population_prognosis
@@ -219,26 +216,25 @@
 
     household_prognosis = PythonOperator(
         task_id="zensus-household-prognosis",
-        python_callable=process_zs.household_prognosis_to_zensus
+        python_callable=process_zs.household_prognosis_to_zensus,
     )
     prognosis_tables >> household_prognosis
     map_zensus_vg250 >> household_prognosis
     demandregio_society >> household_prognosis
     zensus_misc_import >> household_prognosis
 
-
     # Distribute electrical demands to zensus cells
     processed_dr_tables = PythonOperator(
         task_id="create-demand-tables",
-        python_callable=process_dr.create_tables
+        python_callable=process_dr.create_tables,
     )
 
     elec_household_demands_zensus = PythonOperator(
-        task_id="electrical-demands-zensus",
-        python_callable=process_dr.distribute_demands
+        task_id="electrical-household-demands-zensus",
+        python_callable=process_dr.distribute_household_demands,
     )
 
-    setup >> processed_dr_tables >> elec_household_demands_zensus
+    zensus_tables >> processed_dr_tables >> elec_household_demands_zensus
     population_prognosis >> elec_household_demands_zensus
     demandregio_demand_households >> elec_household_demands_zensus
     map_zensus_vg250 >> elec_household_demands_zensus
@@ -305,69 +301,75 @@
     )
 
     create_voronoi = PythonOperator(
-        task_id="create_voronoi",
-        python_callable=substation.create_voronoi
+        task_id="create_voronoi", python_callable=substation.create_voronoi
     )
-    osm_add_metadata  >> substation_tables >> substation_functions
+    osm_add_metadata >> substation_tables >> substation_functions
     substation_functions >> hvmv_substation_extraction >> create_voronoi
     substation_functions >> ehv_substation_extraction >> create_voronoi
     vg250_clean_and_prepare >> hvmv_substation_extraction
     vg250_clean_and_prepare >> ehv_substation_extraction
 
     # osmTGmod ehv/hv grid model generation
     run_osmtgmod = PythonOperator(
-        task_id= "run_osmtgmod",
-        python_callable= osmtgmod.run_osmtgmod,
+        task_id="run_osmtgmod",
+        python_callable=osmtgmod.run_osmtgmod,
     )
 
-
     osmtgmod_pypsa = PythonOperator(
         task_id="osmtgmod_pypsa",
-        python_callable = osmtgmod.osmtgmmod_to_pypsa,
+        python_callable=osmtgmod.osmtgmmod_to_pypsa,
     )
 
     ehv_substation_extraction >> run_osmtgmod
     hvmv_substation_extraction >> run_osmtgmod
     run_osmtgmod >> osmtgmod_pypsa
 
-
     # Import potential areas for wind onshore and ground-mounted PV
     download_re_potential_areas = PythonOperator(
         task_id="download_re_potential_area_data",
         python_callable=re_potential_areas.download_datasets,
     )
     create_re_potential_areas_tables = PythonOperator(
         task_id="create_re_potential_areas_tables",
-        python_callable=re_potential_areas.create_tables
+        python_callable=re_potential_areas.create_tables,
     )
     insert_re_potential_areas = PythonOperator(
         task_id="insert_re_potential_areas",
-        python_callable=re_potential_areas.insert_data
+        python_callable=re_potential_areas.insert_data,
     )
     setup >> download_re_potential_areas >> create_re_potential_areas_tables
     create_re_potential_areas_tables >> insert_re_potential_areas
 
     # Future heat demand calculation based on Peta5_0_1 data
     heat_demand_import = PythonOperator(
         task_id="import-heat-demand",
-        python_callable=import_hd.future_heat_demand_data_import
+        python_callable=import_hd.future_heat_demand_data_import,
     )
     vg250_clean_and_prepare >> heat_demand_import
     zensus_inside_ger_metadata >> heat_demand_import
 
     # Power plant setup
     power_plant_tables = PythonOperator(
         task_id="create-power-plant-tables",
-        python_callable=power_plants.create_tables
+        python_callable=power_plants.create_tables,
     )
 
     power_plant_import = PythonOperator(
         task_id="import-power-plants",
-        python_callable=power_plants.insert_power_plants
+        python_callable=power_plants.insert_power_plants,
     )
     setup >> power_plant_tables >> power_plant_import
     nep_insert_data >> power_plant_import
     retrieve_mastr_data >> power_plant_import
 
+    # Districute electrical CTS demands to zensus grid
 
+    elec_cts_demands_zensus = PythonOperator(
+        task_id="electrical-cts-demands-zensus",
+        python_callable=process_dr.distribute_cts_demands,
+    )
 
+    processed_dr_tables >> elec_cts_demands_zensus
+    heat_demand_import >> elec_cts_demands_zensus
+    demandregio_demand_cts_ind >> elec_cts_demands_zensus
+    map_zensus_vg250 >> elec_cts_demands_zensus

src/egon/data/datasets.yml

@@ -52,9 +52,9 @@ zensus_misc:
   processed:
     schema: "society"
     path_table_map:
-      "csv_Haushalte_100m_Gitter.zip": "destatis_zensus_household_per_ha"
-      "csv_Gebaeude_100m_Gitter.zip": "destatis_zensus_building_per_ha"
-      "csv_Wohnungen_100m_Gitter.zip": "destatis_zensus_apartment_per_ha"
+      "csv_Haushalte_100m_Gitter.zip": "egon_destatis_zensus_household_per_ha"
+      "csv_Gebaeude_100m_Gitter.zip": "egon_destatis_zensus_building_per_ha"
+      "csv_Wohnungen_100m_Gitter.zip": "egon_destatis_zensus_apartment_per_ha"
 
 map_zensus_vg250:
   sources:
@@ -136,12 +136,32 @@ electrical_demands_households:
       table: 'egon_population_prognosis'
     map_zensus_vg250:
       schema: 'boundaries'
-      table: 'egon_map_zensus_nuts3'
+      table: 'egon_map_zensus_vg250'
   targets:
     household_demands_zensus:
       schema: 'demand'
       table: 'egon_demandregio_zensus_electricity'
 
+electrical_demands_cts:
+  sources:
+    demandregio:
+      schema: 'demand'
+      table: 'egon_demandregio_cts_ind'
+      scenarios: ["eGon2035", "eGon100RE"]
+    demandregio_wz:
+      schema: 'demand'
+      table: 'egon_demandregio_wz'
+    heat_demand_cts:
+      schema: 'demand'
+      table: 'egon_peta_heat'
+    map_zensus_vg250:
+      schema: 'boundaries'
+      table: 'egon_map_zensus_vg250'
+  targets:
+    cts_demands_zensus:
+      schema: 'demand'
+      table: 'egon_demandregio_zensus_electricity'
+
 hvmv_substation:
   original_data:
     source:
@@ -187,7 +207,7 @@ society_prognosis:
       table: 'destatis_zensus_population_per_ha'
     zensus_households:
       schema: 'society'
-      table: 'destatis_zensus_household_per_ha'
+      table: 'egon_destatis_zensus_household_per_ha'
     demandregio_population:
       schema: 'society'
       table: 'egon_demandregio_population'

src/egon/data/importing/heat_demand_data/__init__.py

@@ -499,9 +499,14 @@ def adjust_residential_heat_to_zensus(scenario):
     """
     Adjust residential heat demands to fit to zensus population.
 
+    In some cases, Peta assigns residential heat demand to unpopulated cells.
+    This can be caused by the different population data used in Peta or
+    buildings in zenus cells without a population
+    (see :func:`egon.data.importing.zensus.adjust_zensus_misc`)
+
     Residential heat demand in cells without zensus population is dropped.
     Residential heat demand in cells with zensus population is scaled to meet
-    the formal overall residential heat demands.
+    the overall residential heat demands.
 
     Parameters
     ----------

src/egon/data/importing/zensus/__init__.py

@@ -410,3 +410,40 @@ def zensus_misc_to_postgres():
                     FOREIGN KEY (zensus_population_id)
                     REFERENCES {population_table}(id);"""
         )
+
+    # Delete entries for unpopulated cells
+    adjust_zensus_misc()
+
+
+def adjust_zensus_misc():
+    """Deletes zensus households, buildings and aparments in unpopulated cells
+
+    Some unpopulated zensus cells are listed in the table of households,
+    buildings and/or aparments. This can be caused by missing population
+    information due to privacy or other special cases (e.g. holiday homes
+    are listed as buildings but are not permanently populated.)
+    In the follwong tasks of egon-data, only data of populated cells is used.
+
+    Returns
+    -------
+    None.
+
+    """
+    # Get information from data configuration file
+    data_config = egon.data.config.datasets()
+    zensus_population_processed = data_config["zensus_population"]["processed"]
+    zensus_misc_processed = data_config["zensus_misc"]["processed"]
+
+    population_table = (
+        f"{zensus_population_processed['schema']}"
+        f".{zensus_population_processed['table']}"
+    )
+
+    for input_file, table in zensus_misc_processed["path_table_map"].items():
+        db.execute_sql(
+             f"""
+             DELETE FROM {zensus_population_processed['schema']}.{table} as b
+             WHERE b.zensus_population_id IN (
+                 SELECT id FROM {population_table}
+                 WHERE population < 0);"""
+            )