Merge branch 'dev' into features/#162-district-heating-areas

CHANGELOG.rst

@@ -80,6 +80,8 @@ Added
   `#177 <https://github.com/openego/eGon-data/issues/177>`_
 * Extract landuse areas from OSM 
   `#214 <https://github.com/openego/eGon-data/issues/214>`_
+* Integrate weather data and renewable feedin timeseries
+  `#19 <https://github.com/openego/eGon-data/issues/19>`_
 
 .. _PR #159: https://github.com/openego/eGon-data/pull/159
 
@@ -131,3 +133,5 @@ Bug fixes
   `#204 <https://github.com/openego/eGon-data/issues/204>`_
 * Replace wrong table name in SQL function used in substation extraction
   `#236 <https://github.com/openego/eGon-data/issues/236>`_
+* Fix osmtgmod for osm data from 2021 by updating substation in Garenfeld 
+  `#241 <https://github.com/openego/eGon-data/issues/241>`_

README.rst

@@ -119,6 +119,12 @@ packages are required too. Right now these are:
   On recent Ubuntu version you can install it via
   :code:`sudo apt install gdal-bin`.
 
+* To download ERA5 weather data you need to register at the CDS
+  registration page and install the CDS API key as descibed
+  `here <https://cds.climate.copernicus.eu/api-how-to>`_
+  You also have to agree on the `terms of use 
+  <https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products>`_
+
 Installation
 ============
 

setup.py

@@ -83,6 +83,8 @@ def read(*names, **kwargs):
     install_requires=[
         # eg: 'aspectlib==1.1.1', 'six>=1.7',
         "apache-airflow>=1.10.14,<2.0",
+        "atlite==0.0.3",
+        "cdsapi",
         "click",
         "geopandas>=0.9.0",
         "importlib-resources",

src/egon/data/airflow/dags/pipeline.py

@@ -12,6 +12,7 @@
 import airflow
 import egon.data.importing.demandregio as import_dr
 import egon.data.importing.demandregio.install_disaggregator as install_dr
+import egon.data.importing.era5 as import_era5
 import egon.data.importing.etrago as etrago
 import egon.data.importing.heat_demand_data as import_hd
 import egon.data.importing.mastr as mastr
@@ -31,7 +32,9 @@
 import egon.data.importing.scenarios as import_scenarios
 import egon.data.importing.industrial_sites as industrial_sites
 import egon.data.processing.loadarea as loadarea
+import egon.data.processing.renewable_feedin as import_feedin
 import egon.data.processing.district_heating_areas as district_heating_areas
+
 from egon.data import db
 
 
@@ -453,6 +456,46 @@
     osm_add_metadata >> landuse_extraction
     vg250_clean_and_prepare >> landuse_extraction
 
+ # Import weather data
+    download_era5 = PythonOperator(
+        task_id="download-weather-data",
+        python_callable=import_era5.download_era5,
+    )
+    scenario_input_import >> download_era5
+
+    create_weather_tables = PythonOperator(
+        task_id="create-weather-tables",
+        python_callable=import_era5.create_tables,
+    )
+    setup >> create_weather_tables
+
+    import_weather_cells = PythonOperator(
+        task_id="insert-weather-cells",
+        python_callable=import_era5.insert_weather_cells,
+    )
+    create_weather_tables >> import_weather_cells
+    download_era5 >> import_weather_cells
+
+    feedin_wind_onshore = PythonOperator(
+        task_id="insert-feedin-wind",
+        python_callable=import_feedin.wind_feedin_per_weather_cell,
+    )
+
+    feedin_pv = PythonOperator(
+        task_id="insert-feedin-pv",
+        python_callable=import_feedin.pv_feedin_per_weather_cell,
+    )
+
+    feedin_solar_thermal = PythonOperator(
+        task_id="insert-feedin-solar-thermal",
+        python_callable=import_feedin.solar_thermal_feedin_per_weather_cell,
+    )
+
+    import_weather_cells >> [feedin_wind_onshore,
+                             feedin_pv, feedin_solar_thermal]
+    vg250_clean_and_prepare >> [feedin_wind_onshore,
+                             feedin_pv, feedin_solar_thermal]
+
     # District heating areas demarcation
     create_district_heating_areas_table = PythonOperator(
         task_id="create-district-heating-areas-table",

src/egon/data/datasets.yml

@@ -359,3 +359,28 @@ landuse:
       table: 'osm_landuse'
       schema: 'openstreetmap'
 
+era5_weather_data:
+  targets:
+    weather_data:
+      path: 'cutouts'
+    weather_cells:
+      schema: 'supply'
+      table: 'egon_era5_weather_cells'
+
+renewable_feedin:
+  sources:
+    era5_weather_data:
+      path: 'cutouts'
+    weather_cells:
+      schema: 'supply'
+      table: 'egon_era5_weather_cells'
+    vg250_lan_union:
+      schema: 'boundaries'
+      table: 'vg250_lan_union'
+    vg250_sta_union:
+      schema: 'boundaries'
+      table: 'vg250_sta_union'
+  targets:
+    feedin_table:
+      schema: 'supply'
+      table: 'egon_era5_renewable_feedin'

src/egon/data/importing/era5/__init__.py

@@ -0,0 +1,136 @@
+"""
+Central module containing all code dealing with importing era5 weather data.
+"""
+
+import atlite
+import os
+from pathlib import Path
+
+import geopandas as gpd
+import egon.data.config
+from egon.data import db
+from egon.data.importing.scenarios import get_sector_parameters
+from sqlalchemy import Column, String, Float, Integer, ARRAY
+from sqlalchemy.ext.declarative import declarative_base
+from geoalchemy2 import Geometry
+# will be later imported from another file ###
+Base = declarative_base()
+
+class EgonEra5Cells(Base):
+    __tablename__ = 'egon_era5_weather_cells'
+    __table_args__ = {'schema': 'supply'}
+    w_id = Column(Integer, primary_key=True)
+    geom = Column(Geometry('POLYGON', 4326))
+    geom_point = Column(Geometry('POINT', 4326))
+
+
+class EgonRenewableFeedIn(Base):
+    __tablename__ = 'egon_era5_renewable_feedin'
+    __table_args__ = {'schema': 'supply'}
+    w_id = Column(Integer, primary_key=True)
+    weather_year = Column(Integer, primary_key=True)
+    carrier = Column(String, primary_key=True)
+    feedin = Column(ARRAY(Float()))
+
+def create_tables():
+
+    db.execute_sql(
+        "CREATE SCHEMA IF NOT EXISTS supply;")
+    engine = db.engine()
+    EgonEra5Cells.__table__.drop(bind=engine, checkfirst=True)
+    EgonEra5Cells.__table__.create(bind=engine, checkfirst=True)
+    EgonRenewableFeedIn.__table__.drop(bind=engine, checkfirst=True)
+    EgonRenewableFeedIn.__table__.create(bind=engine, checkfirst=True)
+
+def import_cutout(boundary='Europe'):
+    """ Import weather data from cutout
+
+    Returns
+    -------
+    cutout : atlite.cutout.Cutout
+        Weather data stored in cutout
+
+    """
+
+    if boundary == 'Europe':
+        xs = slice(-12., 35.1)
+        ys = slice(72., 33.)
+
+    elif boundary == 'Germany':
+        geom_de = gpd.read_postgis(
+            "SELECT geometry as geom FROM boundaries.vg250_sta_bbox",
+            db.engine()).to_crs(4623).geom
+        xs = slice(geom_de.bounds.minx[0], geom_de.bounds.maxx[0])
+        ys = slice(geom_de.bounds.maxy[0], geom_de.bounds.miny[0])
+
+    else:
+        print(
+            f"Boundary {boundary} not defined. "
+            "Choose either 'Europe' or 'Germany'")
+
+    directory = Path(".") / (
+        egon.data.config.datasets()
+        ['era5_weather_data']['targets']['weather_data']['path'])
+
+    weather_year = get_sector_parameters('global', 'eGon2035')['weather_year']
+
+    cutout = atlite.Cutout(
+            f"europe-{str(weather_year)}-era5",
+            cutout_dir = directory.absolute(),
+            module="era5",
+            xs=xs,
+            ys=ys,
+            years=slice(weather_year, weather_year)
+            )
+
+    return cutout
+
+def download_era5():
+    """ Download weather data from era5
+
+    Returns
+    -------
+    None.
+
+    """
+
+    directory = Path(".") / (
+        egon.data.config.datasets()
+        ['era5_weather_data']['targets']['weather_data']['path'])
+
+    if not os.path.exists(directory):
+
+        os.mkdir(directory)
+
+    cutout = import_cutout()
+
+    if not cutout.prepared:
+
+        cutout.prepare(nprocesses=1)
+
+
+def insert_weather_cells():
+    """ Insert weather cells from era5 into database table
+
+    Returns
+    -------
+    None.
+
+    """
+
+    cfg = egon.data.config.datasets()['era5_weather_data']
+
+    cutout = import_cutout()
+
+    df = gpd.GeoDataFrame(
+        {'geom': cutout.grid_cells()}, geometry='geom', crs=4326)
+
+    df.to_postgis(cfg['targets']['weather_cells']['table'],
+                  schema=cfg['targets']['weather_cells']['schema'],
+                  con=db.engine(), if_exists='append')
+
+    db.execute_sql(
+        f"""UPDATE {cfg['targets']['weather_cells']['schema']}.
+        {cfg['targets']['weather_cells']['table']}
+        SET geom_point=ST_Centroid(geom);"""
+        )