Merge branch 'dev' into features/#169-renaming-gid-and-subst_id

CHANGELOG.rst

@@ -185,6 +185,10 @@ Changed
   `#378 <https://github.com/openego/eGon-data/issues/378>`_
 * Remove version columns from eTraGo tables and related code
   `#384 <https://github.com/openego/eGon-data/issues/384>`_
+* Remove country column from scenario capacities table
+  `#391 <https://github.com/openego/eGon-data/issues/391>`_
+* Update version of zenodo download
+  `#397 <https://github.com/openego/eGon-data/issues/397>`_
 
 Bug fixes
 ---------
@@ -217,6 +221,8 @@ Bug fixes
   `#258 <https://github.com/openego/eGon-data/issues/258>`_
 * Fix conflicting docker containers by setting a project name
   `#289 <https://github.com/openego/eGon-data/issues/289>`_
+* Update task insert-nep-data for pandas version 1.3.0
+  `#322 <https://github.com/openego/eGon-data/issues/322>`_
 * Fix versioning conflict with mv_grid_districts
   `#340 <https://github.com/openego/eGon-data/issues/340>`_
 * Set current working directory as java's temp dir when executing osmosis

src/egon/data/datasets.yml

@@ -446,7 +446,7 @@ solar_rooftop:
 data-bundle:
   sources:
     zenodo:
-      deposit_id: 4896527
+      deposit_id: 5211145
   targets:
     file:
       'data_bundle_egon_data.zip'

src/egon/data/datasets/data_bundle/__init__.py

@@ -7,7 +7,8 @@
 
 import egon.data.config
 from egon.data.datasets import Dataset
-
+from pathlib import Path
+import shutil
 
 def download():
     """
@@ -16,6 +17,10 @@ def download():
     ----------
 
     """
+    data_bundle_path = Path(".") / 'data_bundle_egon_data'
+    # Delete folder if it already exists
+    if data_bundle_path.exists() and data_bundle_path.is_dir():
+        shutil.rmtree(data_bundle_path)
     # Get parameters from config and set download URL
     sources = egon.data.config.datasets()["data-bundle"]["sources"]["zenodo"]
     url = f"""https://zenodo.org/record/{sources['deposit_id']}/files/data_bundle_egon_data.zip"""

src/egon/data/datasets/demandregio/__init__.py

@@ -174,7 +174,7 @@ def insert_cts_ind_wz_definitions():
             delimiter=','
         df = pd.read_csv(
             os.path.join(
-                "data_bundle_egon_data/wz_definition/",
+                "data_bundle_egon_data/WZ_definition/",
                 source['wz_definitions'][sector]),
             delimiter=delimiter,
             header=None).rename(

src/egon/data/datasets/scenario_capacities.py

@@ -20,7 +20,6 @@ class EgonScenarioCapacities(Base):
     __tablename__ = 'egon_scenario_capacities'
     __table_args__ = {'schema': 'supply'}
     index = Column(Integer, primary_key=True)
-    country = Column(String(50))
     component = Column(String(25))
     carrier = Column(String(50))
     capacity = Column(Float)
@@ -56,7 +55,7 @@ class ScenarioCapacities(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ScenarioCapacities",
-            version="0.0.0",
+            version="0.0.2",
             dependencies=dependencies,
             tasks=(
                 create_table,
@@ -94,9 +93,12 @@ def create_table():
 
     engine = db.engine()
     db.execute_sql("CREATE SCHEMA IF NOT EXISTS supply;")
+    EgonScenarioCapacities.__table__.drop(bind=engine, checkfirst=True)
+    NEP2021ConvPowerPlants.__table__.drop(bind=engine, checkfirst=True)
     EgonScenarioCapacities.__table__.create(bind=engine, checkfirst=True)
     NEP2021ConvPowerPlants.__table__.create(bind=engine, checkfirst=True)
 
+
 def insert_capacities_per_federal_state_nep():
     """Inserts installed capacities per federal state accordning to
     NEP 2035 (version 2021), scenario 2035 C
@@ -113,7 +115,7 @@ def insert_capacities_per_federal_state_nep():
     # Delete rows if already exist
     db.execute_sql("DELETE FROM supply.egon_scenario_capacities "
                    "WHERE scenario_name = 'eGon2035' "
-                   "AND country = 'Deutschland'")
+                   "AND nuts != 'DE'")
 
     # read-in installed capacities per federal state of germany
     target_file = os.path.join(
@@ -130,7 +132,7 @@ def insert_capacities_per_federal_state_nep():
     # sort NEP-carriers:
     rename_carrier = {'Wind onshore': 'wind_onshore',
                      'Wind offshore': 'wind_offshore',
-                     'Sonstige Konventionelle': 'other_non_renewable',
+                     'sonstige Konventionelle': 'other_non_renewable',
                      'Speicherwasser': 'reservoir',
                      'Laufwasser': 'run_of_river',
                      'Biomasse': 'biomass',
@@ -139,7 +141,7 @@ def insert_capacities_per_federal_state_nep():
                      'PV (Aufdach)': 'solar_rooftop',
                      'PV (Freiflaeche)': 'solar',
                      'Pumpspeicher': 'pumped_hydro',
-                     'Sonstige EE': 'other_renewable',
+                     'sonstige EE': 'other_renewable',
                      'Oel': 'oil',
                      'Haushaltswaermepumpen': 'residential_rural_heat_pump',
                      'KWK < 10 MW': 'small_chp'}
@@ -177,13 +179,11 @@ def insert_capacities_per_federal_state_nep():
 
 
         data['carrier'] = data.index.map(rename_carrier)
-        data = data.groupby(data.carrier).sum().reset_index()
+        data = data.groupby(data.carrier)[bl].sum().reset_index()
         data['component'] = 'generator'
-        data['country'] = 'Deutschland'
         data['nuts'] = map_nuts.nuts[bl]
         data['scenario_name'] = 'eGon2035'
 
-
         # According to NEP, each heatpump has 3kW_el installed capacity
         data.loc[data.carrier == 'residential_rural_heat_pump', bl] *= 3e-6
         data.loc[data.carrier ==
@@ -299,7 +299,7 @@ def district_heating_input():
     """
     # import data to dataframe
     file = os.path.join(
-        "data_bundle_egon_data/nep2035_version2021/",
+        "data_bundle_egon_data/nep2035_version2021",
         scenario_config('eGon2035')['paths']['capacities'])
     df = pd.read_excel(file, sheet_name='Kurzstudie_KWK', dtype={'Wert':float})
     df.set_index(['Energietraeger', 'Name'], inplace=True)
@@ -319,7 +319,7 @@ def district_heating_input():
         entry = EgonScenarioCapacities(
             component = 'link',
             scenario_name = 'eGon2035',
-            country = 'Deutschland',
+            nuts = 'DE',
             carrier = 'urban_central_'+ (
                 'heat_pump' if c=='Grosswaermepumpe' else 'resistive_heater'),
             capacity = df.loc[(c, 'Fernwaermeerzeugung'), 'Wert']*1e6/
@@ -333,7 +333,7 @@ def district_heating_input():
         entry = EgonScenarioCapacities(
         component = 'generator',
         scenario_name = 'eGon2035',
-        country = 'Deutschland',
+        nuts = 'DE',
         carrier = 'urban_central_'+ (
                 'solar_thermal_collector' if c =='Solarthermie'
                                 else 'geo_thermal'),

src/egon/data/datasets/scenario_parameters/__init__.py

@@ -45,7 +45,7 @@ def insert_scenarios():
 
     """
 
-    db.execute_sql("DELETE FROM scenario.egon_scenario_parameters")
+    db.execute_sql("DELETE FROM scenario.egon_scenario_parameters CASCADE;")
 
     session = sessionmaker(bind=db.engine())()
 
@@ -158,7 +158,7 @@ class ScenarioParameters(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ScenarioParameters",
-            version="0.0.0",
+            version="0.0.1",
             dependencies=dependencies,
             tasks=(
                 create_table,