Skip to content

Commit

Permalink
Merge branch 'features/#586-annual-demand-cencus-household-electricit…
Browse files Browse the repository at this point in the history 
…y' into continuous-integration/run-everything-over-the-weekend-v2
  • Loading branch information
@nailend
nailend committed Feb 13, 2022
2 parents 84036d4 + c598d5c commit 6f50824
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 129 deletions.
4 changes: 2 additions & 2 deletions CHANGELOG.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,8 +323,8 @@ Changed
`#651 <https://github.com/openego/eGon-data/issues/#651>`_
* H2 feed in links are changed to non extendable
`#653 <https://github.com/openego/eGon-data/issues/653>`_
* Remove the '_fixed' suffix
`#628 <https://github.com/openego/eGon-data/issues/628>`_
* Fill table demand.egon_demandregio_zensus_electricity after profile allocation
`#620 <https://github.com/openego/eGon-data/issues/586>`_


Bug fixes
Expand Down
80 changes: 37 additions & 43 deletions src/egon/data/airflow/dags/pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from egon.data.datasets.etrago_setup import EtragoSetup
from egon.data.datasets.fill_etrago_gen import Egon_etrago_gen
from egon.data.datasets.gas_aggregation import GasAggregation
from egon.data.datasets.gas_areas import GasAreaseGon2035, GasAreaseGon100RE
from egon.data.datasets.gas_areas import GasAreaseGon100RE, GasAreaseGon2035
from egon.data.datasets.gas_grid import GasNodesandPipes
from egon.data.datasets.gas_prod import CH4Production
from egon.data.datasets.heat_demand import HeatDemandImport
Expand All @@ -47,9 +47,11 @@
HydrogenPowerLinkEtragoeGon100RE,
HydrogenStoreEtrago,
)
from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand
from egon.data.datasets.industrial_gas_demand import IndustrialGasDemandeGon2035
from egon.data.datasets.industrial_gas_demand import IndustrialGasDemandeGon100RE
from egon.data.datasets.industrial_gas_demand import (
IndustrialGasDemand,
IndustrialGasDemandeGon100RE,
IndustrialGasDemandeGon2035,
)
from egon.data.datasets.industrial_sites import MergeIndustrialSites
from egon.data.datasets.industry import IndustrialDemandCurves
from egon.data.datasets.loadarea import LoadArea
Expand Down Expand Up @@ -153,20 +155,6 @@
"osm_buildings_streets.preprocessing"
]

# Distribute household electrical demands to zensus cells
household_electricity_demand_annual = HouseholdElectricityDemand(
dependencies=[
demandregio,
zensus_vg250,
zensus_miscellaneous,
society_prognosis,
]
)

elec_household_demands_zensus = tasks[
"electricity_demand.distribute-household-demands"
]

saltcavern_storage = SaltcavernData(dependencies=[data_bundle, vg250])

# NEP data import
Expand Down Expand Up @@ -233,9 +221,7 @@
heat_demands_abroad_download = tasks["heat_demand_europe.download"]

# Download industrial gas demand
industrial_gas_demand = IndustrialGasDemand(
dependencies=[setup]
)
industrial_gas_demand = IndustrialGasDemand(dependencies=[setup])

# Extract landuse areas from osm data set
load_area = LoadArea(dependencies=[osm, vg250])
Expand Down Expand Up @@ -283,22 +269,7 @@
dependencies=[vg250, mv_grid_districts]
)

# Distribute electrical CTS demands to zensus grid
cts_electricity_demand_annual = CtsElectricityDemand(
dependencies=[
demandregio,
zensus_vg250,
zensus_mv_grid_districts,
heat_demand_Germany,
etrago_input_data,
household_electricity_demand_annual,
]
)

elec_cts_demands_zensus = tasks[
"electricity_demand.distribute-cts-demands"
]

#
mv_hh_electricity_load_2035 = PythonOperator(
task_id="MV-hh-electricity-load-2035",
python_callable=hh_profiles.mv_grid_district_HH_electricity_load,
Expand Down Expand Up @@ -344,6 +315,30 @@
"electricity_demand_timeseries.hh_buildings.map-houseprofiles-to-buildings"
]

# Get household electrical demands for cencus cells
household_electricity_demand_annual = HouseholdElectricityDemand(
dependencies=[map_houseprofiles_to_buildings]
)

elec_annual_household_demands_cells = tasks[
"electricity_demand.get-annual-household-el-demand-cells"
]

# Distribute electrical CTS demands to zensus grid
cts_electricity_demand_annual = CtsElectricityDemand(
dependencies=[
demandregio,
zensus_vg250,
zensus_mv_grid_districts,
heat_demand_Germany,
etrago_input_data,
household_electricity_demand_annual,
]
)

elec_cts_demands_zensus = tasks[
"electricity_demand.distribute-cts-demands"
]
# Industry

industrial_sites = MergeIndustrialSites(
Expand Down Expand Up @@ -437,10 +432,7 @@
)

insert_power_to_h2_installations_100RE = HydrogenPowerLinkEtragoeGon100RE(
dependencies=[
insert_power_to_h2_installations,
insert_h2_grid
]
dependencies=[insert_power_to_h2_installations, insert_h2_grid]
)

# Create gas voronoi eGon100RE
Expand All @@ -454,7 +446,9 @@
)

# CH4 storages import
insert_data_ch4_storages = CH4Storages(dependencies=[create_gas_polygons_egon2035])
insert_data_ch4_storages = CH4Storages(
dependencies=[create_gas_polygons_egon2035]
)

# Assign industrial gas demand eGon2035
assign_industrial_gas_demand = IndustrialGasDemandeGon2035(
Expand Down Expand Up @@ -518,7 +512,7 @@

feedin_pv >> solar_rooftop_etrago
elec_cts_demands_zensus >> solar_rooftop_etrago
elec_household_demands_zensus >> solar_rooftop_etrago
elec_annual_household_demands_cells >> solar_rooftop_etrago
etrago_input_data >> solar_rooftop_etrago
map_zensus_grid_districts >> solar_rooftop_etrago

Expand Down
175 changes: 91 additions & 84 deletions src/egon/data/datasets/electricity_demand/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,35 @@
data from demandRegio
"""
import egon.data.config
from sqlalchemy import Column, Float, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
import pandas as pd

from egon.data import db
from egon.data.datasets.electricity_demand.temporal import insert_cts_load
from egon.data.datasets import Dataset
from sqlalchemy import Column, String, Float, Integer, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from egon.data.datasets.electricity_demand.temporal import insert_cts_load
from egon.data.datasets.electricity_demand_timeseries.hh_buildings import (
HouseholdElectricityProfilesOfBuildings,
get_iee_hh_demand_profiles_raw,
)
from egon.data.datasets.electricity_demand_timeseries.hh_profiles import (
HouseholdElectricityProfilesInCensusCells,
)
from egon.data.datasets.zensus_vg250 import DestatisZensusPopulationPerHa
import egon.data.config

# will be later imported from another file ###
Base = declarative_base()
engine = db.engine()


class HouseholdElectricityDemand(Dataset):
def __init__(self, dependencies):
super().__init__(
name="HouseholdElectricityDemand",
version="0.0.2",
version="0.0.3",
dependencies=dependencies,
tasks=(create_tables, distribute_household_demands),
tasks=(create_tables, get_annual_household_el_demand_cells),
)


Expand Down Expand Up @@ -62,101 +72,98 @@ def create_tables():
)


def distribute_household_demands():
""" Distribute electrical demands for households to zensus cells.
The demands on nuts3-level from demandregio are linear distributed
to the share of future population in each zensus cell.
Returns
-------
None.
def get_annual_household_el_demand_cells():
"""
Annual electricity demand per cell is determined
sources = egon.data.config.datasets()["electrical_demands_households"][
"sources"
]
Timeseries for every cell are accumulated, the maximum value
determined and with the respective nuts3 factor scaled for 2035 and 2050
scenario.
target = egon.data.config.datasets()["electrical_demands_households"][
"targets"
]["household_demands_zensus"]
Note
----------
In test-mode 'SH' the iteration takes place by 'cell_id' to avoid
intensive RAM usage. For whole Germany 'nuts3' are taken and
RAM > 32GB is necessary.
"""

db.execute_sql(
f"""DELETE FROM {target['schema']}.{target['table']}
WHERE sector = 'residential'"""
)
with db.session_scope() as session:
cells_query = (
session.query(
HouseholdElectricityProfilesOfBuildings,
HouseholdElectricityProfilesInCensusCells.nuts3,
HouseholdElectricityProfilesInCensusCells.factor_2035,
HouseholdElectricityProfilesInCensusCells.factor_2050,
)
.filter(
HouseholdElectricityProfilesOfBuildings.cell_id
== HouseholdElectricityProfilesInCensusCells.cell_id
)
.order_by(HouseholdElectricityProfilesOfBuildings.id)
)

# Select match between zensus cells and nuts3 regions of vg250
map_nuts3 = db.select_dataframe(
f"""SELECT zensus_population_id, vg250_nuts3 as nuts3 FROM
{sources['map_zensus_vg250']['schema']}.
{sources['map_zensus_vg250']['table']}""",
index_col="zensus_population_id",
df_buildings_and_profiles = pd.read_sql(
cells_query.statement, cells_query.session.bind, index_col="id"
)

# Insert data per scenario
for scn in sources["demandregio"]["scenarios"]:
# Read demand profiles from egon-data-bundle
df_profiles = get_iee_hh_demand_profiles_raw()

# Set target years per scenario
if scn == "eGon2035":
year = 2035
elif scn == "eGon100RE":
year = 2050
else:
print(f"Warning: Scenario {scn} can not be imported.")

# Select prognosed population per zensus cell
zensus = db.select_dataframe(
f"""SELECT * FROM
{sources['population_prognosis_zensus']['schema']}.
{sources['population_prognosis_zensus']['table']}
WHERE year = {year}
AND population > 0""",
index_col="zensus_population_id",
)
def ve(s):
raise (ValueError(s))

# Add nuts3 key to zensus cells
zensus["nuts3"] = map_nuts3.nuts3
dataset = egon.data.config.settings()["egon-data"]["--dataset-boundary"]
iterate_over = (
"nuts3"
if dataset == "Everything"
else "cell_id"
if dataset == "Schleswig-Holstein"
else ve(f"'{dataset}' is not a valid dataset boundary.")
)

# Calculate share of nuts3 population per zensus cell
zensus["population_share"] = zensus.population.groupby(
zensus.nuts3
).apply(lambda grp: grp / grp.sum())
df_annual_demand = pd.DataFrame(
columns=["eGon2035", "eGon100RE", "zensus_population_id"]
)

# Select forecasted electrical demands from demandregio table
demand_nuts3 = db.select_dataframe(
f"""SELECT nuts3, SUM(demand) as demand FROM
{sources['demandregio']['schema']}.
{sources['demandregio']['table']}
WHERE scenario = '{scn}'
GROUP BY nuts3""",
index_col="nuts3",
for _, df in df_buildings_and_profiles.groupby(by=iterate_over):
df_annual_demand_iter = pd.DataFrame(
columns=["eGon2035", "eGon100RE", "zensus_population_id"]
)

# Scale demands on nuts3 level linear to population share
zensus["demand"] = zensus["population_share"].mul(
demand_nuts3.demand[zensus["nuts3"]].values
df_annual_demand_iter["eGon2035"] = (
df_profiles.loc[:, df["profile_id"]].sum(axis=0)
* df["factor_2035"].values
)

# Set scenario name and sector
zensus["scenario"] = scn
zensus["sector"] = "residential"

# Rename index
zensus.index = zensus.index.rename("zensus_population_id")

# Insert data to target table
zensus[["scenario", "demand", "sector"]].to_sql(
target["table"],
schema=target["schema"],
con=db.engine(),
if_exists="append",
df_annual_demand_iter["eGon100RE"] = (
df_profiles.loc[:, df["profile_id"]].sum(axis=0)
* df["factor_2050"].values
)
df_annual_demand_iter["zensus_population_id"] = df["cell_id"].values
df_annual_demand = df_annual_demand.append(df_annual_demand_iter)

df_annual_demand = (
df_annual_demand.groupby("zensus_population_id").sum().reset_index()
)
df_annual_demand["sector"] = "residential"
df_annual_demand = df_annual_demand.melt(
id_vars=["zensus_population_id", "sector"],
var_name="scenario",
value_name="demand",
)
# convert from Wh to MWh
df_annual_demand["demand"] = df_annual_demand["demand"] / 1e6

# Insert data to target table
df_annual_demand.to_sql(
name=EgonDemandRegioZensusElectricity.__table__.name,
schema=EgonDemandRegioZensusElectricity.__table__.schema,
con=db.engine(),
index=False,
if_exists="append",
)


def distribute_cts_demands():
""" Distribute electrical demands for cts to zensus cells.
"""Distribute electrical demands for cts to zensus cells.
The demands on nuts3-level from demandregio are linear distributed
to the heat demand of cts in each zensus cell.
Expand Down

0 comments on commit 6f50824

Please sign in to comment.