Merge branch 'dev' into features/#436-H2-and-CH4-carriers

CHANGELOG.rst

@@ -204,6 +204,8 @@ Changed
   `#397 <https://github.com/openego/eGon-data/issues/397>`_
 * Rename columns gid to id
   `#169 <https://github.com/openego/eGon-data/issues/169>`_
+* Remove upper version limit of pandas
+  `#383 <https://github.com/openego/eGon-data/issues/383>`_
 * Use random seed from CLI parameters for CHP and society prognosis functions
   `#351 <https://github.com/openego/eGon-data/issues/351>`_
 * Changed demand.egon_schmidt_industrial_sites - table and merged table (industrial_sites)
@@ -212,6 +214,9 @@ Changed
   `#436 <https://github.com/openego/eGon-data/issues/436>`_
 * Adjust file path for industrial sites import
   `#397 <https://github.com/openego/eGon-data/issues/418>`_
+* Rename columns subst_id to bus_id
+  `#335 <https://github.com/openego/eGon-data/issues/335>`_
+
 
 Bug fixes
 ---------
@@ -270,3 +275,6 @@ Bug fixes
   `#432 <https://github.com/openego/eGon-data/issues/432>`_
 * Add curl to prerequisites in the docs
   `#440 <https://github.com/openego/eGon-data/issues/440>`_
+* Replace NAN by 0 to avoid empty p_set column in DB
+  `#414 <https://github.com/openego/eGon-data/issues/414>`_
+

setup.py

@@ -93,7 +93,7 @@ def read(*names, **kwargs):
         "netcdf4",
         "oedialect==0.0.8",
         "openpyxl",
-        "pandas>1.2.0,<1.3.0",
+        "pandas>1.2.0",
         "psycopg2",
         "pyaml",
         "rasterio",

src/egon/data/datasets/electricity_demand/__init__.py

@@ -16,7 +16,7 @@ class HouseholdElectricityDemand(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="HouseholdElectricityDemand",
-            version="0.0.1",
+            version="0.0.2",
             dependencies=dependencies,
             tasks=(create_tables,
                    distribute_household_demands)
@@ -26,7 +26,7 @@ class CtsElectricityDemand(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="CtsElectricityDemand",
-            version="0.0.1",
+            version="0.0.2",
             dependencies=dependencies,
             tasks=(distribute_cts_demands,
                    insert_cts_load)

src/egon/data/datasets/electricity_demand/temporal.py

@@ -16,7 +16,7 @@ class EgonEtragoElectricityCts(Base):
     __tablename__ = "egon_etrago_electricity_cts"
     __table_args__ = {"schema": "demand"}
 
-    subst_id = Column(Integer, primary_key=True)
+    bus_id = Column(Integer, primary_key=True)
     scn_name = Column(String, primary_key=True)
     p_set = Column(ARRAY(Float))
     q_set = Column(ARRAY(Float))
@@ -113,7 +113,7 @@ def calc_load_curves_cts(scenario):
     Returns
     -------
     pandas.DataFrame
-        Demand timeseries of cts per substation id
+        Demand timeseries of cts per bus id
 
     """
 
@@ -138,7 +138,7 @@ def calc_load_curves_cts(scenario):
     demands_zensus = db.select_dataframe(
             f"""SELECT a.zensus_population_id, a.demand,
             b.vg250_nuts3 as nuts3,
-            c.subst_id
+            c.bus_id
             FROM {sources['zensus_electricity']['schema']}.
             {sources['zensus_electricity']['table']} a
             INNER JOIN
@@ -168,10 +168,10 @@ def calc_load_curves_cts(scenario):
 
     # Calculate shares of cts branches per hvmv substation
     share_subst = demands_zensus.drop(
-        'demand', axis=1).groupby('subst_id').mean()
+        'demand', axis=1).groupby('bus_id').mean()
 
     # Calculate cts annual demand per hvmv substation
-    annual_demand_subst = demands_zensus.groupby('subst_id').demand.sum()
+    annual_demand_subst = demands_zensus.groupby('bus_id').demand.sum()
 
     # Return electrical load curves per hvmv substation
     return calc_load_curve(share_subst, annual_demand_subst)

src/egon/data/datasets/electricity_demand_etrago.py

@@ -29,11 +29,11 @@ def demands_per_bus(scenario):
 
     # Select data on CTS electricity demands per bus
     cts_curves = db.select_dataframe(
-        f"""SELECT subst_id, p_set FROM
+        f"""SELECT bus_id, p_set FROM
                 {sources['cts_curves']['schema']}.
                 {sources['cts_curves']['table']}
                 WHERE scn_name = '{scenario}'""",
-        index_col="subst_id",
+        index_col="bus_id",
     )
 
     # Rename index
@@ -187,7 +187,7 @@ class ElectricalLoadEtrago(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="Electrical_load_etrago",
-            version="0.0.1",
+            version="0.0.2",
             dependencies=dependencies,
             tasks=(export_to_db,),
         )

src/egon/data/datasets/heat_etrago/__init__.py

@@ -228,7 +228,7 @@ class HeatEtrago(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="HeatEtrago",
-            version="0.0.2",
+            version="0.0.3",
             dependencies=dependencies,
             tasks=(buses, supply),
         )

src/egon/data/datasets/heat_etrago/power_to_heat.py

@@ -301,7 +301,7 @@ def assign_electrical_bus(heat_pumps, multiple_per_mv_grid=False):
     # Select mv grid distrcits
     mv_grid_district = db.select_geodataframe(
         f"""
-        SELECT subst_id, geom FROM
+        SELECT bus_id, geom FROM
         {sources['egon_mv_grid_district']['schema']}.
         {sources['egon_mv_grid_district']['table']}
         """
@@ -339,7 +339,7 @@ def assign_electrical_bus(heat_pumps, multiple_per_mv_grid=False):
     # Assign power bus per zensus cell
     cells["power_bus"] = gpd.sjoin(
         cells, mv_grid_district, how="inner", op="intersects"
-    ).subst_id
+    ).bus_id
 
     # Calclate district heating demand per substaion
     demand_per_substation = pd.DataFrame(

src/egon/data/datasets/heat_supply/__init__.py

@@ -134,7 +134,7 @@ class HeatSupply(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="HeatSupply",
-            version="0.0.1",
+            version="0.0.2",
             dependencies=dependencies,
             tasks=(create_tables,
                 district_heating, individual_heating, potential_germany),

src/egon/data/datasets/heat_supply/individual_heating.py

@@ -124,7 +124,7 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True):
     # Select residential heat demand per mv grid district and federal state
     heat_per_mv = db.select_geodataframe(
         f"""
-        SELECT d.subst_id as bus_id, SUM(demand) as demand,
+        SELECT d.bus_id as bus_id, SUM(demand) as demand,
         c.vg250_lan as state, d.geom
         FROM {sources['heat_demand']['schema']}.
         {sources['heat_demand']['table']} a
@@ -133,17 +133,17 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True):
         ON a.zensus_population_id = b.zensus_population_id
         JOIN {sources['map_vg250_grid']['schema']}.
         {sources['map_vg250_grid']['table']} c
-        ON b.subst_id = c.bus_id
+        ON b.bus_id = c.bus_id
         JOIN {sources['mv_grids']['schema']}.
         {sources['mv_grids']['table']} d
-        ON d.subst_id = c.bus_id
+        ON d.bus_id = c.bus_id
         WHERE scenario = '{scenario}'
         AND sector = 'residential'
         AND a.zensus_population_id NOT IN (
             SELECT zensus_population_id
             FROM {sources['map_dh']['schema']}.{sources['map_dh']['table']}
             WHERE scenario = '{scenario}')
-        GROUP BY d.subst_id, vg250_lan, geom
+        GROUP BY d.bus_id, vg250_lan, geom
         """,
         index_col = 'bus_id')
 
@@ -191,7 +191,7 @@ def plot_heat_supply(resulting_capacities):
     mv_grids = db.select_geodataframe(
         """
         SELECT * FROM grid.egon_mv_grid_district
-        """, index_col='subst_id')
+        """, index_col='bus_id')
 
     for c in ['CHP', 'heat_pump']:
         mv_grids[c] = resulting_capacities[

src/egon/data/datasets/hh_demand_profiles.py

@@ -226,7 +226,7 @@ class EgonEtragoElectricityHouseholds(Base):
     __table_args__ = {"schema": "demand"}
 
     version = Column(String, primary_key=True)
-    subst_id = Column(Integer, primary_key=True)
+    bus_id = Column(Integer, primary_key=True)
     scn_name = Column(String, primary_key=True)
     p_set = Column(ARRAY(Float))
     q_set = Column(ARRAY(Float))
@@ -235,7 +235,7 @@ class EgonEtragoElectricityHouseholds(Base):
 hh_demand_setup = partial(
     Dataset,
     name="HH Demand",
-    version="0.0.1",
+    version="0.0.2",
     dependencies=[],
     # Tasks are declared in pipeline as function is used multiple times with different args
     # To differentiate these tasks PythonOperator with specific id-names are used
@@ -1451,15 +1451,15 @@ def mv_grid_district_HH_electricity_load(
     Returns
     -------
     pd.DataFrame
-        Multiindexed dataframe with `timestep` and `subst_id` as indexers.
+        Multiindexed dataframe with `timestep` and `bus_id` as indexers.
         Demand is given in kWh.
     """
     engine = db.engine()
 
     with db.session_scope() as session:
         cells_query = session.query(
             HouseholdElectricityProfilesInCensusCells,
-            MapZensusGridDistricts.subst_id,
+            MapZensusGridDistricts.bus_id,
         ).join(
             MapZensusGridDistricts,
             HouseholdElectricityProfilesInCensusCells.cell_id
@@ -1481,7 +1481,7 @@ def mv_grid_district_HH_electricity_load(
 
     # Create aggregated load profile for each MV grid district
     mvgd_profiles_dict = {}
-    for grid_district, data in cells.groupby("subst_id"):
+    for grid_district, data in cells.groupby("bus_id"):
         mvgd_profile = get_load_timeseries(
             df_profiles=df_profiles,
             df_cell_demand_metadata=data,
@@ -1494,7 +1494,7 @@ def mv_grid_district_HH_electricity_load(
 
     # Reshape data: put MV grid ids in columns to a single index column
     mvgd_profiles = mvgd_profiles.reset_index()
-    mvgd_profiles.columns = ["subst_id", "p_set"]
+    mvgd_profiles.columns = ["bus_id", "p_set"]
 
     # Add remaining columns
     mvgd_profiles["version"] = version

src/egon/data/datasets/industry/__init__.py

@@ -351,7 +351,7 @@ class IndustrialDemandCurves(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="Industrial_demand_curves",
-            version="0.0.3",
+            version="0.0.4",
             dependencies=dependencies,
             tasks=(
                 create_tables,

src/egon/data/datasets/industry/temporal.py

@@ -43,7 +43,7 @@ def identify_bus(load_curves, demand_area):
 
     # Select mv griddistrict
     griddistrict = db.select_geodataframe(
-        f"""SELECT subst_id, geom FROM
+        f"""SELECT bus_id, geom FROM
                 {sources['egon_mv_grid_district']['schema']}.
                 {sources['egon_mv_grid_district']['table']}""",
         geom_col="geom",
@@ -86,7 +86,7 @@ def identify_bus(load_curves, demand_area):
     # Combine dataframes to bring loadcurves and bus id together
     curves_da = pd.merge(
         load_curves.T,
-        peak_bus[["subst_id", "id"]],
+        peak_bus[["bus_id", "id"]],
         left_index=True,
         right_on="id",
     )
@@ -165,7 +165,7 @@ def calc_load_curves_ind_osm(scenario):
     curves_da = identify_bus(load_curves, demand_area)
 
     # Group all load curves per bus
-    curves_bus = curves_da.drop(["id"], axis=1).groupby("subst_id").sum()
+    curves_bus = curves_da.drop(["id"], axis=1).fillna(0).groupby("bus_id").sum()
 
     # Initalize pandas.DataFrame for export to database
     load_ts_df = pd.DataFrame(index=curves_bus.index, columns=["p_set"])
@@ -288,7 +288,7 @@ def calc_load_curves_ind_sites(scenario):
 
     # Group all load curves per bus and wz
     curves_bus = (
-        curves_da.groupby(["subst_id", "wz"]).sum().drop(["id"], axis=1)
+        curves_da.fillna(0).groupby(["bus_id", "wz"]).sum().drop(["id"], axis=1)
     )
 
     # Initalize pandas.DataFrame for pf table load timeseries