Merge branch 'dev' into features/#611-add-dc-lines-to-SE-DK

CHANGELOG.rst

@@ -295,9 +295,14 @@ Changed
   `#545 <https://github.com/openego/eGon-data/issues/545>`_
 * Integrate fuel and CO2 costs for eGon2035 to scenario parameters
   `#549 <https://github.com/openego/eGon-data/issues/549>`_
+* Fix RE potential areas outside of Germany by updating
+  the dataset. Import files from data bundle.
+  `#592 <https://github.com/openego/eGon-data/issues/592>`_
+  `#595 <https://github.com/openego/eGon-data/issues/595>`_
 * Add DC lines from Germany to Sweden and Denmark
   `#611 <https://github.com/openego/eGon-data/issues/611>`_
 
+
 Bug fixes
 ---------
 * Heat demand data import
@@ -379,3 +384,9 @@ Bug fixes
   `#585 <https://github.com/openego/eGon-data/issues/585>`_
 * Change source for H2 steel tank storage to Danish Energy Agency
   `#605 <https://github.com/openego/eGon-data/issues/605>`_
+* Change carrier name from 'pv' to 'solar' in eTraGo_generators
+  `#617 <https://github.com/openego/eGon-data/issues/617>`_
+* Assign "carrier" to transmission lines with no value in grid.egon_etrago_line
+  `#625 <https://github.com/openego/eGon-data/issues/625>`_
+* Fix deleting from eTraGo tables
+  `#613 <https://github.com/openego/eGon-data/issues/613>`_

CONTRIBUTING.rst

@@ -323,7 +323,8 @@ For previous discussions on metadata, you may want to check
 Helpers
 ^^^^^^^
 
-You can use the `Metadata creator <https://meta.rl-institut.de>`_ **GUI**.
+You can use the
+`Metadata creator <https://meta.rl-institut.de/meta_creator/141>`_ **GUI**.
 Fill the fields and hit `Edit JSON` to get the metadata string. Vice versa,
 you can paste a metadata string into this box and the fields will be filled
 automatically which may be helpful if you want to amend existing strings.

src/egon/data/airflow/dags/pipeline.py

@@ -211,7 +211,9 @@
     ]
 
     # Import potential areas for wind onshore and ground-mounted PV
-    re_potential_areas = re_potential_area_setup(dependencies=[setup])
+    re_potential_areas = re_potential_area_setup(
+        dependencies=[setup, data_bundle]
+    )
     re_potential_areas.insert_into(pipeline)
 
     # Future heat demand calculation based on Peta5_0_1 data

src/egon/data/datasets.yml

@@ -280,23 +280,16 @@ mastr:
   deposit_id: 808086
 
 re_potential_areas:
-  original_data:
-    source:
-      url:
-        #TODO: finally replace sandbox
-        - "https://sandbox.zenodo.org/record/746695/files/potentialarea_pv_agriculture.gpkg?download=1"
-        - "https://sandbox.zenodo.org/record/746695/files/potentialarea_pv_road_railway.gpkg?download=1"
-        - "https://sandbox.zenodo.org/record/746695/files/potentialarea_wind.gpkg?download=1"
-      url_testmode:
-        - "https://sandbox.zenodo.org/record/746695/files/potentialarea_pv_agriculture_SH.gpkg?download=1"
-        - "https://sandbox.zenodo.org/record/746695/files/potentialarea_pv_road_railway_SH.gpkg?download=1"
-        - "https://sandbox.zenodo.org/record/746695/files/potentialarea_wind_SH.gpkg?download=1"
-    target:
-      schema: "supply"
-      path_table_map:
-        "potentialarea_pv_agriculture.gpkg": "egon_re_potential_area_pv_agriculture"
-        "potentialarea_pv_road_railway.gpkg": "egon_re_potential_area_pv_road_railway"
-        "potentialarea_wind.gpkg": "egon_re_potential_area_wind"
+  target:
+    schema: "supply"
+    path_table_map:
+      "potentialarea_pv_agriculture.gpkg": "egon_re_potential_area_pv_agriculture"
+      "potentialarea_pv_road_railway.gpkg": "egon_re_potential_area_pv_road_railway"
+      "potentialarea_wind.gpkg": "egon_re_potential_area_wind"
+    path_table_map_testmode:
+      "potentialarea_pv_agriculture_SH.gpkg": "egon_re_potential_area_pv_agriculture"
+      "potentialarea_pv_road_railway_SH.gpkg": "egon_re_potential_area_pv_road_railway"
+      "potentialarea_wind_SH.gpkg": "egon_re_potential_area_wind"
 
 peta5_0_1_res_heat_demands:
   original_data:
@@ -504,7 +497,7 @@ solar_rooftop:
 data-bundle:
   sources:
     zenodo:
-      deposit_id: 5743452
+      deposit_id: 5846441
   targets:
     file:
       'data_bundle_egon_data.zip'
@@ -550,7 +543,7 @@ heat_supply:
       schema: 'demand'
       table: 'egon_map_zensus_district_heating_areas'
     etrago_buses:
-      table: 'egon_pf_hv_bus'
+      table: 'egon_etrago_bus'
       schema: 'grid'
   targets:
     district_heating_supply:
@@ -726,6 +719,9 @@ electrical_load_curves_industry:
 
 etrago_electricity:
   sources:
+    etrago_buses:
+      table: 'egon_etrago_bus'
+      schema: 'grid'
     osm_curves:
       schema: 'demand'
       table: 'egon_osm_ind_load_curves'
@@ -756,7 +752,7 @@ chp_location:
     egon_mv_grid_district: 'grid.egon_mv_grid_district'
     ehv_voronoi: "grid.egon_ehv_substation_voronoi"
     etrago_buses:
-      table: 'egon_pf_hv_bus'
+      table: 'egon_etrago_bus'
       schema: 'grid'
     osm_landuse:
       table: 'osm_landuse'
@@ -791,7 +787,7 @@ chp_etrago:
       schema: 'demand'
       table: 'egon_district_heating_areas'
     etrago_buses:
-      table: 'egon_pf_hv_bus'
+      table: 'egon_etrago_bus'
       schema: 'grid'
   targets:
     link:
@@ -840,6 +836,9 @@ DSM_CTS_industry:
 
 generators_etrago:
   sources:
+    bus:
+      schema: 'grid'
+      table: 'egon_etrago_bus'
     power_plants:
       schema: 'supply'
       table: 'egon_power_plants'

src/egon/data/datasets/chp_etrago.py

@@ -14,7 +14,7 @@ class ChpEtrago(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ChpEtrago",
-            version="0.0.2",
+            version="0.0.3",
             dependencies=dependencies,
             tasks=(insert),
         )
@@ -41,6 +41,16 @@ def insert():
         DELETE FROM {targets['link']['schema']}.{targets['link']['table']}
         WHERE carrier LIKE '%%CHP%%'
         AND scn_name = 'eGon2035'
+        AND bus0 IN 
+        (SELECT bus_id 
+         FROM {sources['etrago_buses']['schema']}.{sources['etrago_buses']['table']}
+         WHERE scn_name = 'eGon2035'
+         AND country = 'DE')
+        AND bus1 IN 
+        (SELECT bus_id 
+         FROM {sources['etrago_buses']['schema']}.{sources['etrago_buses']['table']}
+         WHERE scn_name = 'eGon2035'
+         AND country = 'DE')
         """
     )
     db.execute_sql(

src/egon/data/datasets/electrical_neighbours.py

@@ -470,7 +470,7 @@ def central_transformer(scenario, sources, targets, central_buses, new_lines):
         DELETE FROM {targets['transformers']['schema']}.
         {targets['transformers']['table']}
         WHERE scn_name = '{scenario}'
-        AND trafo_id IN (
+        AND trafo_id NOT IN (
             SELECT branch_id
             FROM {sources['osmtgmod_branch']['schema']}.
             {sources['osmtgmod_branch']['table']}

src/egon/data/datasets/electricity_demand_etrago.py

@@ -71,7 +71,8 @@ def demands_per_bus(scenario):
 
     # Create one df by appending all imported dataframes
 
-    demand_curves = cts_curves.append([ind_curves_osm, ind_curves_sites, hh_curves])
+    demand_curves = cts_curves.append(
+        [ind_curves_osm, ind_curves_sites, hh_curves])
 
     # Split array to single columns in the dataframe
     demand_curves_split = demand_curves
@@ -102,7 +103,7 @@ def export_to_db():
     None.
 
     """
-
+    sources = egon.data.config.datasets()["etrago_electricity"]["sources"]
     targets = egon.data.config.datasets()["etrago_electricity"]["targets"]
 
     for scenario in ["eGon2035", "eGon100RE"]:
@@ -114,6 +115,13 @@ def export_to_db():
             {targets['etrago_load']['schema']}.{targets['etrago_load']['table']}
             WHERE scn_name = '{scenario}'
             AND carrier = 'AC'
+            AND bus IN (
+                SELECT bus_id FROM 
+                {sources['etrago_buses']['schema']}.
+                {sources['etrago_buses']['table']}
+                WHERE country = 'DE'
+                AND carrier = 'AC'
+                AND scn_name = '{scenario}')
             """
         )
 
@@ -197,7 +205,7 @@ class ElectricalLoadEtrago(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="Electrical_load_etrago",
-            version="0.0.4",
+            version="0.0.5",
             dependencies=dependencies,
             tasks=(export_to_db,),
         )

src/egon/data/datasets/fill_etrago_gen.py

@@ -6,11 +6,12 @@
 from egon.data.datasets import Dataset
 import egon.data.config
 
+
 class Egon_etrago_gen(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="etrago_generators",
-            version="0.0.3",
+            version="0.0.4",
             dependencies=dependencies,
             tasks=(fill_etrago_generators,),
         )
@@ -22,7 +23,7 @@ def fill_etrago_generators():
     cfg = egon.data.config.datasets()["generators_etrago"]
 
     # Delete power plants from previous iterations of this script
-    delete_previuos_gen(cfg)    
+    delete_previuos_gen(cfg)
 
     # Load required tables
     (
@@ -33,8 +34,8 @@ def fill_etrago_generators():
         pp_time,
     ) = load_tables(con, cfg)
 
-    power_plants = adjust_power_plants_table(
-        power_plants=power_plants, renew_feedin=renew_feedin, cfg=cfg
+    renew_feedin = adjust_renew_feedin_table(
+        renew_feedin=renew_feedin, cfg=cfg
     )
 
     etrago_pp = group_power_plants(
@@ -80,7 +81,7 @@ def group_power_plants(power_plants, renew_feedin, etrago_gen_orig, cfg):
         func=agg_func
     )
     etrago_pp = etrago_pp.reset_index(drop=True)
-    
+
     if np.isnan(etrago_gen_orig["generator_id"].max()):
         max_id = 0
     else:
@@ -122,7 +123,7 @@ def fill_etrago_gen_time_table(
     ]
 
     etrago_pp_time = etrago_pp_time[
-        (etrago_pp_time["carrier"] == "pv")
+        (etrago_pp_time["carrier"] == "solar")
         | (etrago_pp_time["carrier"] == "wind_onshore")
         | (etrago_pp_time["carrier"] == "wind_offshore")
     ]
@@ -154,10 +155,10 @@ def fill_etrago_gen_time_table(
         con=con,
         if_exists="append",
     )
-    return etrago_pp
+    return etrago_pp_time
 
 
-def load_tables(con, cfg, scenario='eGon2035'):
+def load_tables(con, cfg, scenario="eGon2035"):
     sql = f"""
     SELECT * FROM
     {cfg['sources']['power_plants']['schema']}.
@@ -219,15 +220,16 @@ def power_timeser(weather_data):
         return -1
 
 
-def adjust_power_plants_table(power_plants, renew_feedin, cfg):
-    # Define carrier 'solar' as 'pv'
-    carrier_pv_mask = power_plants["carrier"] == "solar"
-    power_plants.loc[carrier_pv_mask, "carrier"] = "pv"
+def adjust_renew_feedin_table(renew_feedin, cfg):
+
+    # Define carrier 'pv' as 'solar'
+    carrier_pv_mask = renew_feedin["carrier"] == "pv"
+    renew_feedin.loc[carrier_pv_mask, "carrier"] = "solar"
 
     # convert renewable feedin lists to arrays
     renew_feedin["feedin"] = renew_feedin["feedin"].apply(np.array)
 
-    return power_plants
+    return renew_feedin
 
 
 def delete_previuos_gen(cfg):
@@ -238,47 +240,40 @@ def delete_previuos_gen(cfg):
                    WHERE carrier <> 'CH4' AND carrier <> 'solar_rooftop'
                    AND carrier <> 'solar_thermal_collector'
                    AND carrier <> 'geo_thermal'
+                   AND bus IN (
+                       SELECT bus_id FROM {cfg['sources']['bus']['schema']}.
+                       {cfg['sources']['bus']['table']}
+                       WHERE country = 'DE'
+                       AND carrier = 'AC')
                    """
     )
 
 
 def set_timeseries(power_plants, renew_feedin):
     def timeseries(pp):
-        try:
-            if pp.weather_cell_id != -1:
-                feedin_time = renew_feedin[
-                    (renew_feedin["w_id"] == pp.weather_cell_id)
-                    & (renew_feedin["carrier"] == pp.carrier)
-                ].feedin.iloc[0]
-                return feedin_time
-            else:
-                df = power_plants[
-                    (power_plants["bus_id"] == pp.bus_id)
-                    & (power_plants["carrier"] == pp.carrier)
-                ]
-                total_int_cap = df.el_capacity.sum()
-                df["feedin"] = 0
-                df["feedin"] = df.apply(
-                    lambda x: renew_feedin[
-                        (renew_feedin["w_id"] == x.weather_cell_id)
-                        & (renew_feedin["carrier"] == x.carrier)
-                    ].feedin.iloc[0],
-                    axis=1,
-                )
-                df["feedin"] = df.apply(
-                    lambda x: x.el_capacity / total_int_cap * x.feedin, axis=1
-                )
-                return df.feedin.sum()
-        #######################################################################
-        ####################### DELETE THIS EXCEPTION #########################
-        except:
+        if pp.weather_cell_id != -1:
+            feedin_time = renew_feedin[
+                (renew_feedin["w_id"] == pp.weather_cell_id)
+                & (renew_feedin["carrier"] == pp.carrier)
+            ].feedin.iloc[0]
+            return feedin_time
+        else:
             df = power_plants[
                 (power_plants["bus_id"] == pp.bus_id)
                 & (power_plants["carrier"] == pp.carrier)
             ]
-            return list(df.weather_cell_id)
+            total_int_cap = df.el_capacity.sum()
+            df["feedin"] = 0
+            df["feedin"] = df.apply(
+                lambda x: renew_feedin[
+                    (renew_feedin["w_id"] == x.weather_cell_id)
+                    & (renew_feedin["carrier"] == x.carrier)
+                ].feedin.iloc[0],
+                axis=1,
+            )
+            df["feedin"] = df.apply(
+                lambda x: x.el_capacity / total_int_cap * x.feedin, axis=1
+            )
+            return df.feedin.sum()
 
-    ####################### DELETE THIS EXCEPTION #############################
-    ###########################################################################
     return timeseries
-