Merge branch 'dev' into features/#962-final-buildings-mapping

CHANGELOG.rst

@@ -219,8 +219,14 @@ Added
   `#962 <https://github.com/openego/eGon-data/issues/962>`_
 * Add charging infrastructure for e-mobility
   `#937 <https://github.com/openego/eGon-data/issues/937>`_
+* Add zipfile check
+  `#969 <https://github.com/openego/eGon-data/issues/969>`_
 * Add marginal costs for generators abroad and for carriers nuclear and coal
   `#907 <https://github.com/openego/eGon-data/issues/907>`_
+* Add wind off shore power plants for eGon100RE
+  `#868 <https://github.com/openego/eGon-data/issues/868>`_
+* Write simBEV metadata to DB table
+  `PR #978 <https://github.com/openego/eGon-data/pull/978>`_
 
 .. _PR #159: https://github.com/openego/eGon-data/pull/159
 .. _PR #703: https://github.com/openego/eGon-data/pull/703
@@ -411,15 +417,24 @@ Changed
   `#866 <https://github.com/openego/eGon-data/issues/866>`_
 * Add noflex scenario for motorized individual travel
   `#821 <https://github.com/openego/eGon-data/issues/821>`_
+* Allocate PV home batteries to mv grid districts
+  `#749 <https://github.com/openego/eGon-data/issues/749>`_
 * Add sanity checks for motorized individual travel
   `#820 <https://github.com/openego/eGon-data/issues/820>`_
 * Parallelize sanity checks
   `#882 <https://github.com/openego/eGon-data/issues/882>`_
+* Insert crossboarding gas pipeline with Germany in eGon100RE
+  `#881 <https://github.com/openego/eGon-data/issues/881>`_
 * Rename noflex to lowflex scenario for motorized individual travel
   `#921 <https://github.com/openego/eGon-data/issues/921>`_
 * Update creation of heat demand timeseries
   `#857 <https://github.com/openego/eGon-data/issues/857>`_
   `#856 <https://github.com/openego/eGon-data/issues/856>`_
+* Introduce carrier name 'others'
+  `#819 <https://github.com/openego/eGon-data/issues/819>`_
+* Add rural heat pumps per medium voltage grid district
+  `#987 <https://github.com/openego/eGon-data/issues/987>`_
+
 
 Bug Fixes
 ---------
@@ -611,7 +626,10 @@ Bug Fixes
   `#914 <https://github.com/openego/eGon-data/issues/914>`_
 * Drop era5 weather cell table using cascade
   `#909 <https://github.com/openego/eGon-data/issues/909>`_
-
+* Delete gas bus with wrong country code
+  `#958 <https://github.com/openego/eGon-data/issues/958>`_
+* Overwrite capacities for conventional power plants with data from nep list
+  `#403 <https://github.com/openego/eGon-data/issues/403>`_
 
 .. _PR #692: https://github.com/openego/eGon-data/pull/692
 .. _#343: https://github.com/openego/eGon-data/issues/343

README.rst

@@ -194,8 +194,18 @@ If running :code:`egon-data` results in an error, we also have collected
 a list of `known runtime errors`_, which can consult in search of a
 solution.
 
+To run the workflow from the CLI without using :code:`egon-data serve` you can use
+
+.. code-block:: bash
+
+   egon-data airflow scheduler
+   egon-data airflow dags trigger egon-data-processing-pipeline
+
+For further details how to use the CLI see `Apache Airflow CLI Reference`_.
+
 .. _Apache Airflow: https://airflow.apache.org/docs/apache-airflow/stable/ui.html#ui-screenshots
 .. _known runtime errors: https://eGon-data.readthedocs.io/en/latest/troubleshooting.html#runtime-errors
+.. _Apache Airflow CLI Reference: https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html
 
 .. warning::
 

setup.py

@@ -5,8 +5,7 @@
 from glob import glob
 from os.path import basename, dirname, join, splitext
 import io
-import re
-
+import re 
 from setuptools import find_packages, setup
 
 

src/egon/data/airflow/dags/pipeline.py

@@ -369,14 +369,6 @@
         ]
     )
 
-    # Gas abroad
-    gas_abroad_insert_data = GasNeighbours(
-        dependencies=[
-            gas_grid_insert_data,
-            foreign_lines,
-        ]
-    )
-
     # Insert hydrogen buses
     insert_hydrogen_buses = HydrogenBusEtrago(
         dependencies=[
@@ -420,6 +412,17 @@
         dependencies=[create_gas_polygons_egon2035, insert_h2_grid, vg250]
     )
 
+    # Gas abroad
+    gas_abroad_insert_data = GasNeighbours(
+        dependencies=[
+            gas_grid_insert_data,
+            run_pypsaeursec,
+            foreign_lines,
+            insert_hydrogen_buses,
+            create_gas_polygons_egon100RE
+        ]
+    )
+
     # Import gas production
     gas_production_insert_data = CH4Production(
         dependencies=[create_gas_polygons_egon2035]

src/egon/data/datasets.yml

@@ -351,6 +351,7 @@ power_plants:
       mastr_location: "location_elec_generation_raw.csv"
       mastr_combustion_without_chp: "supply.egon_mastr_conventional_without_chp"
       mastr_storage: "bnetza_mastr_storage_cleaned.csv"
+      mastr_gsgk: "bnetza_mastr_gsgk_cleaned.csv"
       capacities: "supply.egon_scenario_capacities"
       geom_germany: "boundaries.vg250_sta_union"
       geom_federal_states: "boundaries.vg250_lan"
@@ -373,6 +374,9 @@ storages:
       egon_mv_grid_district: "grid.egon_mv_grid_district"
       ehv_voronoi: "grid.egon_ehv_substation_voronoi"
       nep_conv: "supply.egon_nep_2021_conventional_powerplants"
+      nep_capacities: "NEP2035_V2021_scnC2035.xlsx"
+      generators: "grid.egon_etrago_generator"
+      bus: "grid.egon_etrago_bus"
   target:
       table: 'egon_storages'
       schema: 'supply'

src/egon/data/datasets/electrical_neighbours.py

@@ -725,15 +725,15 @@ def map_carriers_tyndp():
         "Nuclear": "nuclear",
         "Offshore Wind": "wind_offshore",
         "Onshore Wind": "wind_onshore",
-        "Other non-RES": "other_non_renewable",
-        "Other RES": "other_renewable",
+        "Other non-RES": "others",
+        "Other RES": "others",
         "P2G": "power_to_gas",
         "PS Closed": "pumped_hydro",
         "PS Open": "reservoir",
         "Reservoir": "reservoir",
         "Run-of-River": "run_of_river",
         "Solar PV": "solar",
-        "Solar Thermal": "other_renewable",
+        "Solar Thermal": "others",
         "Waste": "Other RES",
     }
 
@@ -914,11 +914,10 @@ def insert_generators(capacities):
     gen = capacities[
         capacities.carrier.isin(
             [
-                "other_non_renewable",
+                "others",
                 "wind_offshore",
                 "wind_onshore",
                 "solar",
-                "other_renewable",
                 "reservoir",
                 "run_of_river",
                 "lignite",

src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py

@@ -84,15 +84,16 @@
 
 from egon.data import db, subprocess
 from egon.data.datasets import Dataset
-from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (
+from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (  # noqa: E501
     EgonEvCountMunicipality,
     EgonEvCountMvGridDistrict,
     EgonEvCountRegistrationDistrict,
+    EgonEvMetadata,
     EgonEvMvGridDistrict,
     EgonEvPool,
     EgonEvTrip,
 )
-from egon.data.datasets.emobility.motorized_individual_travel.ev_allocation import (
+from egon.data.datasets.emobility.motorized_individual_travel.ev_allocation import (  # noqa: E501
     allocate_evs_numbers,
     allocate_evs_to_grid_districts,
 )
@@ -105,11 +106,12 @@
     TRIP_COLUMN_MAPPING,
     WORKING_DIR,
 )
-from egon.data.datasets.emobility.motorized_individual_travel.model_timeseries import (
+from egon.data.datasets.emobility.motorized_individual_travel.model_timeseries import (  # noqa: E501
     delete_model_data_from_db,
     generate_model_data_bunch,
     generate_model_data_eGon100RE_remaining,
     generate_model_data_eGon2035_remaining,
+    read_simbev_metadata_file,
 )
 
 
@@ -152,6 +154,8 @@ def create_tables():
     EgonEvTrip.__table__.create(bind=engine, checkfirst=True)
     EgonEvMvGridDistrict.__table__.drop(bind=engine, checkfirst=True)
     EgonEvMvGridDistrict.__table__.create(bind=engine, checkfirst=True)
+    EgonEvMetadata.__table__.drop(bind=engine, checkfirst=True)
+    EgonEvMetadata.__table__.create(bind=engine, checkfirst=True)
 
     # Create dir for results, if it does not exist
     result_dir = WORKING_DIR / Path("results")
@@ -357,6 +361,41 @@ def import_csv(f):
         os.remove(trip_file)
 
 
+def write_metadata_to_db():
+    """
+    Write used SimBEV metadata per scenario to database.
+    """
+    dtypes = {
+        "scenario": str,
+        "eta_cp": float,
+        "stepsize": int,
+        "start_date": np.datetime64,
+        "end_date": np.datetime64,
+        "soc_min": float,
+        "grid_timeseries": bool,
+        "grid_timeseries_by_usecase": bool,
+    }
+
+    for scenario_name in ["eGon2035", "eGon100RE"]:
+        meta_run_config = read_simbev_metadata_file(
+            scenario_name, "config"
+        ).loc["basic"]
+
+        meta_run_config = (
+            meta_run_config.to_frame()
+            .T.assign(scenario=scenario_name)[dtypes.keys()]
+            .astype(dtypes)
+        )
+
+        meta_run_config.to_sql(
+            name=EgonEvMetadata.__table__.name,
+            schema=EgonEvMetadata.__table__.schema,
+            con=db.engine(),
+            if_exists="append",
+            index=False,
+        )
+
+
 class MotorizedIndividualTravel(Dataset):
     def __init__(self, dependencies):
         def generate_model_data_tasks(scenario_name):
@@ -414,13 +453,20 @@ def generate_model_data_tasks(scenario_name):
 
         super().__init__(
             name="MotorizedIndividualTravel",
-            version="0.0.4",
+            version="0.0.6",
             dependencies=dependencies,
             tasks=(
                 create_tables,
                 {
-                    (download_and_preprocess, allocate_evs_numbers),
-                    (extract_trip_file, write_evs_trips_to_db),
+                    (
+                        download_and_preprocess,
+                        allocate_evs_numbers,
+                    ),
+                    (
+                        extract_trip_file,
+                        write_metadata_to_db,
+                        write_evs_trips_to_db,
+                    ),
                 },
                 allocate_evs_to_grid_districts,
                 delete_model_data_from_db,

src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py

@@ -2,8 +2,11 @@
 DB tables / SQLAlchemy ORM classes for motorized individual travel
 """
 
-from sqlalchemy import (  # ForeignKeyConstraint,
+from sqlalchemy import (
+    Boolean,
     Column,
+    DateTime,
+    Float,
     ForeignKey,
     Integer,
     SmallInteger,
@@ -232,3 +235,19 @@ class EgonEvMvGridDistrict(Base):
     egon_ev_pool_ev_id = Column(Integer, nullable=False)
 
     # ev = relationship("EgonEvPool", back_populates="mvgds")
+
+
+class EgonEvMetadata(Base):
+    """List of EV Pool Metadata"""
+
+    __tablename__ = "egon_ev_metadata"
+    __table_args__ = {"schema": "demand"}
+
+    scenario = Column(String, primary_key=True)
+    eta_cp = Column(Float)
+    stepsize = Column(Integer)
+    start_date = Column(DateTime)
+    end_date = Column(DateTime)
+    soc_min = Column(Float)
+    grid_timeseries = Column(Boolean)
+    grid_timeseries_by_usecase = Column(Boolean)

src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py

@@ -37,7 +37,7 @@
 import pandas as pd
 
 from egon.data import db
-from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (
+from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (  # noqa: E501
     EgonEvMvGridDistrict,
     EgonEvPool,
     EgonEvTrip,
@@ -60,8 +60,6 @@
 )
 from egon.data.datasets.mv_grid_districts import MvGridDistricts
 
-# from egon.data.datasets.scenario_parameters import get_sector_parameters
-
 
 def data_preprocessing(
     scenario_data: pd.DataFrame, ev_data_df: pd.DataFrame
@@ -93,7 +91,7 @@ def data_preprocessing(
     # charging capacity in MVA
     ev_data_df = ev_data_df.assign(
         charging_capacity_grid_MW=(
-            ev_data_df.charging_capacity_grid / 10 ** 3
+            ev_data_df.charging_capacity_grid / 10**3
         ),
         minimum_charging_time=(
             ev_data_df.charging_demand
@@ -619,7 +617,7 @@ def write_link(scenario_name: str) -> None:
                         carrier="BEV charger",
                         efficiency=float(run_config.eta_cp),
                         p_nom=(
-                            load_time_series_df.simultaneous_plugged_in_charging_capacity.max()
+                            load_time_series_df.simultaneous_plugged_in_charging_capacity.max()  # noqa: E501
                         ),
                         p_nom_extendable=False,
                         p_nom_min=0,
@@ -641,7 +639,7 @@ def write_link(scenario_name: str) -> None:
                         temp_id=1,
                         p_min_pu=None,
                         p_max_pu=(
-                            hourly_load_time_series_df.ev_availability.to_list()
+                            hourly_load_time_series_df.ev_availability.to_list()  # noqa: E501
                         ),
                     )
                 )
@@ -744,7 +742,7 @@ def write_load(
                 scenario_name=scenario_name,
                 connection_bus_id=emob_bus_id,
                 load_ts=(
-                    hourly_load_time_series_df.driving_load_time_series.to_list()
+                    hourly_load_time_series_df.driving_load_time_series.to_list()  # noqa: E501
                 ),
             )
         else: