Merge branch 'dev' into features/#169-renaming-gid-and-subst_id

CHANGELOG.rst

@@ -124,10 +124,12 @@ Added
   `#237 <https://github.com/openego/eGon-data/issues/237>`_
 * Merge electrical loads per bus and export to etrago tables
   `#328 <https://github.com/openego/eGon-data/issues/328>`_
+* Insert industial gas demand
+  `#321 <https://github.com/openego/eGon-data/issues/358>`_
 * Integrate existing CHP and extdended CHP > 10MW_el
   `#266 <https://github.com/openego/eGon-data/issues/266>`_
-
-.. _PR #159: https://github.com/openego/eGon-data/pull/159
+* Add random seed to CLI parameters
+  `#351 <https://github.com/openego/eGon-data/issues/351>`_
 
 
 Changed
@@ -239,5 +241,9 @@ Bug fixes
   `#387 <https://github.com/openego/eGon-data/issues/390>`_
 * Add xlrd and openpyxl to installation setup
   `#400 <https://github.com/openego/eGon-data/issues/400>`_
+* Remove hard-coded slashes in file paths to ensure Windows compatibility
+  `#398 <https://github.com/openego/eGon-data/issues/398>`_
+* Add missing dependency in pipeline.py
+  `#412 <https://github.com/openego/eGon-data/issues/412>`_
 
 

src/egon/data/airflow/dags/pipeline.py

@@ -37,6 +37,7 @@
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
 from egon.data.datasets.zensus_vg250 import ZensusVg250
 from egon.data.datasets.gas_prod import GasProduction
+from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand
 import airflow
 
 import egon.data.importing.zensus as import_zs
@@ -236,19 +237,6 @@
     heat_demand_Germany = HeatDemandImport(
         dependencies=[vg250, scenario_parameters, zensus_vg250])
 
-    # Distribute electrical CTS demands to zensus grid
-    cts_electricity_demand_annual = CtsElectricityDemand(
-        dependencies=[
-            demandregio,
-            zensus_vg250,
-            heat_demand_Germany,
-            etrago_input_data,
-            household_electricity_demand_annual,
-        ]
-    )
-
-    elec_cts_demands_zensus = tasks[
-        'electricity_demand.distribute-cts-demands']
 
     # Gas grid import
     gas_grid_insert_data = PythonOperator(
@@ -281,6 +269,10 @@
     gas_production_insert_data = GasProduction(
         dependencies=[create_gas_polygons])
 
+    # Insert industrial gas demand
+    industrial_gas_demand = IndustrialGasDemand( 
+     dependencies=[create_gas_polygons]) 
+
     # Extract landuse areas from osm data set
     create_landuse_table = PythonOperator(
         task_id="create-landuse-table",
@@ -331,6 +323,22 @@
     )
 
     map_zensus_grid_districts = tasks["zensus_mv_grid_districts.mapping"]
+
+    # Distribute electrical CTS demands to zensus grid
+    cts_electricity_demand_annual = CtsElectricityDemand(
+        dependencies=[
+            demandregio,
+            zensus_vg250,
+            heat_demand_Germany,
+            etrago_input_data,
+            household_electricity_demand_annual,
+            zensus_mv_grid_districts
+        ]
+    )
+
+    elec_cts_demands_zensus = tasks[
+        'electricity_demand.distribute-cts-demands']
+
 
     # Map federal states to mv_grid_districts
     vg250_mv_grid_districts = Vg250MvGridDistricts(

src/egon/data/cli.py

@@ -142,6 +142,19 @@
     show_default=True,
 )
 
+@click.option(
+    "--random-seed",
+    default=42,
+    metavar="RANDOM_SEED",
+    help=(
+        "Random seed used by some tasks in the pipeline to ensure "
+        " deterministic behaviour. All published results in the eGon project "
+        " will be created with the default value so keep it if you want to "
+        " make sure to get the same results."
+    ),
+    show_default=True,
+)
+
 @click.version_option(version=egon.data.__version__)
 @click.pass_context
 def egon_data(context, **kwargs):

src/egon/data/datasets/demandregio/__init__.py

@@ -2,7 +2,6 @@
 adjusting data from demandRegio
 
 """
-import os
 import pandas as pd
 import numpy as np
 import egon.data.config
@@ -14,6 +13,7 @@
 from egon.data.datasets.demandregio.install_disaggregator import (
     clone_and_install)
 from egon.data.datasets import Dataset
+from pathlib import Path
 
 try:
     from disaggregator import data, spatial, config
@@ -40,7 +40,7 @@ class DemandRegio(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="DemandRegio",
-            version="0.0.0",
+            version="0.0.1",
             dependencies=dependencies,
             tasks=(clone_and_install, create_tables,
                    insert_society_data, insert_household_demand,
@@ -168,14 +168,19 @@ def insert_cts_ind_wz_definitions():
     engine = db.engine()
 
     for sector in source['wz_definitions']:
+
+        file_path = (
+            Path(".") /
+            "data_bundle_egon_data" /
+            "WZ_definition" /
+            source['wz_definitions'][sector])
+
         if sector=='CTS':
             delimiter=';'
         else:
             delimiter=','
         df = pd.read_csv(
-            os.path.join(
-                "data_bundle_egon_data/WZ_definition/",
-                source['wz_definitions'][sector]),
+            file_path,
             delimiter=delimiter,
             header=None).rename(
                 {0: 'wz', 1: 'definition'},
@@ -233,10 +238,14 @@ def adjust_cts_ind_nep(ec_cts_ind, sector):
     sources = (egon.data.config.datasets()
                ['demandregio_cts_ind_demand']['sources'])
 
+    file_path = (
+        Path(".") /
+        "data_bundle_egon_data" /
+        "nep2035_version2021" /
+        sources['new_consumers_2035'])
+
     # get data from NEP per federal state
-    new_con = pd.read_csv(os.path.join(
-        "data_bundle_egon_data/nep2035_version2021/",
-        sources['new_consumers_2035']),
+    new_con = pd.read_csv(file_path,
         delimiter=';', decimal=',', index_col=0)
 
     # match nuts3 regions to federal states

src/egon/data/datasets/gas_prod.py

@@ -2,18 +2,18 @@
 """
 The central module containing all code dealing with importing gas production data
 """
-import os
+#import os
 import ast
 import pandas as pd
 import geopandas as gpd
 import numpy as np
 import geopandas
 
 from egon.data import db
-from egon.data.importing.gas_grid import next_id
 from egon.data.config import settings
 from egon.data.datasets import Dataset
 from urllib.request import urlretrieve
+from pathlib import Path
 
 class GasProduction(Dataset):
      def __init__(self, dependencies):
@@ -34,9 +34,12 @@ def load_NG_generators():
         Dataframe containing the natural gas producion units in Germany
 
     """
-    target_file = os.path.join(
-        "datasets/gas_data/",
-        'data/IGGIELGN_Productions.csv')
+    target_file = (
+            Path(".") /
+            "datasets" /
+            "gas_data" /
+            "data" /
+            "IGGIELGN_Productions.csv")
 
     NG_generators_list = pd.read_csv(target_file,
                                delimiter=';', decimal='.',
@@ -100,7 +103,11 @@ def load_biogas_generators():
     # Download file
     basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
     url = "https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/" + basename
-    target_file = "datasets/gas_data/" + basename
+    target_file = (
+            Path(".") /
+            "datasets" /
+            "gas_data" /
+            basename)
 
     urlretrieve(url, target_file)
 
@@ -210,7 +217,7 @@ def import_gas_generators():
     )
 
     # Select next id value
-    new_id = next_id('generator')
+    new_id = db.next_etrago_id('generator')
 
     CH4_generators_list = pd.concat([load_NG_generators(), load_biogas_generators()])
     CH4_generators_list['generator_id'] = range(new_id, new_id + len(CH4_generators_list))

src/egon/data/datasets/heat_supply/geothermal.py

@@ -9,6 +9,7 @@
 import geopandas as gpd
 import pandas as pd
 import numpy as np
+from pathlib import Path
 
 from egon.data import db, config
 
@@ -24,8 +25,12 @@ def calc_geothermal_potentials():
         data={'NDB': 35, 'ORG': 90, 'SMB': 125}, name = 'm_flow')
 
     ## geothermal potentials per temperature (p. 94)
-    potentials = gpd.read_file(
-        'data_bundle_egon_data/geothermal_potential/geothermal_potential_germany.shp')
+    file_path = (
+        Path(".") /
+        "data_bundle_egon_data" /
+        "geothermal_potential" /
+        "geothermal_potential_germany.shp")
+    potentials = gpd.read_file(file_path)
     ## temperature heating system in °C (p. 95)
     sys_temp = 60
     ## temeprature losses heat recuperator in °C (p. 95)