PyPSA · fneum · Sep 5, 2025 · Sep 5, 2025 · Sep 5, 2025 · Sep 5, 2025
diff --git a/config/config.default.yaml b/config/config.default.yaml
@@ -14,6 +14,10 @@ remote:
   ssh: ""
   path: ""
 
+secrets:
+  entsoe_token: ""
+  corine: ""
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
   prefix: ""
@@ -381,7 +385,7 @@ transformers:
 load:
   fill_gaps:
     enable: true
-    interpolate_limit: 3
+    interpolate_limit: 6
     time_shift_for_large_gaps: 1w
   manual_adjustments: true
   scaling_factor: 1.0
@@ -1055,6 +1059,15 @@ data:
   synthetic_electricity_demand:
     source: primary
     version: latest
+  opsd_electricity_demand:
+    source: primary
+    version: latest
+  entsoe_electricity_demand:
+    source: primary
+    version: latest
+  neso_electricity_demand:
+    source: primary
+    version: latest
   copernicus_land_cover:
     source: primary
     version: latest
@@ -1153,9 +1166,6 @@ overpass_api:
     email: contact@pypsa.org
     website: https://github.com/PyPSA/pypsa-eur
 
-secrets:
-  corine: '' #Add API key here if primary source is used for retrieving corine dataset after registering in https://land.copernicus.eu/user/login
-
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving
 solving:
   options:

diff --git a/data/versions.csv b/data/versions.csv
@@ -44,6 +44,9 @@
 "scigrid_gas","archive","1.1.0","['deprecated', 'not-supported']","https://zenodo.org/records/4642569/files/IGGIELGN.zip",""
 "synthetic_electricity_demand","primary","v2","['latest', 'supported']","https://zenodo.org/records/10820928/files/demand_hourly.csv","the primary is already from Zenodo published by original authors"
 "synthetic_electricity_demand","archive","0.1.0","['deprecated', 'might-work']","https://zenodo.org/records/7070438/files/demand_hourly.csv",""
+"opsd_electricity_demand","primary","unknown","['latest', 'supported']","https://data.open-power-system-data.org/time_series/","multiple versions concatenated"
+"entsoe_electricity_demand","primary","unknown","['latest', 'supported']","https://transparency.entsoe.eu/load/total/dayAhead","API request using entsoe-py package"
+"neso_electricity_demand","primary","unknown","['latest', 'supported']","https://www.neso.energy/data-portal/historic-demand-data",""
 "copernicus_land_cover","primary","v3.0.1","['latest', 'supported']","https://zenodo.org/records/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif","The primary is already from Zenodo, documentation in https://zenodo.org/records/4723921"
 "copernicus_land_cover","archive","v2.0.2","['deprecated', 'might-work']","https://zenodo.org/records/3939038/files/PROBAV_LC100_global_v3.0.1_2015-base_Discrete-Classification-map_EPSG-4326.tif",""
 "ship_raster","primary","v5","['latest', 'supported']","https://datacatalogfiles.worldbank.org/ddh-published/0037580/5/DR0045406/shipdensity_global.zip",""

diff --git a/doc/configtables/data.csv b/doc/configtables/data.csv
@@ -50,6 +50,15 @@ scigrid_gas,,,
 synthetic_electricity_demand,,,
 -- source,str,"{primary, archive}","Source of the Synthetic Electricity Demand data."
 -- version,str,,"Version of the Synthetic Electricity Demand data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source."
+opsd_electricity_demand,,,
+-- source,str,"{primary}","Source of the Open Power System Data Electricity Demand data."
+-- version,str,,"Version of the Open Power System Data Electricity Demand data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source."
+entsoe_electricity_demand,,,
+-- source,str,"{primary}","Source of the ENTSO-E Electricity Demand data."
+-- version,str,,"Version of the ENTSO-E Electricity Demand data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source."
+neso_electricity_demand,,,
+-- source,str,"{primary}","Source of the NESO Electricity Demand data."
+-- version,str,,"Version of the NESO Electricity Demand data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source."
 copernicus_land_cover,,,
 -- source,str,"{primary, archive}","Source of the Copernicus Land Cover data."
 -- version,str,,"Version of the Copernicus Land Cover data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source."

diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv
@@ -9,6 +9,7 @@ remote,,,
 -- path,--,,"Optionally specify the file path within the remote cluster to be synchronized."
 secrets,,,
 -- corine,--,,"API token for corine dataset retrieval. You can also pass the token by setting the environment variable `CORINE_API_TOKEN`. See `scripts/retrieve_corine_dataset_primary.py` for more instructions."
+-- entsoe_token,--,,API token for ENTSOE data retrieval.
 overpass_api,,,
 -- url,--,string,"Overpass API endpoint URL. See `https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances`_ for available public instances."
 -- max_tries,--,integer,"Maximum retry attempts for Overpass API requests. Please be respectful to the Overpass API fair use policy of the individual instances."

diff --git a/doc/data_inventory.csv b/doc/data_inventory.csv
@@ -13,6 +13,9 @@
 "jrc_idees","JRC-IDEES-2021","The JRC-IDEES-2021 release contains a consistent set of disaggregated energy-economy-emissions data for each Member State of the European Union, covering all sectors of the energy system for the 2000-2021 period: industry, buildings, transport, and power generation.", "European Commission Joint Research Centre", "https://data.jrc.ec.europa.eu/dataset/82322924-506a-4c9a-8532-2bdd30d69bf5", "CC-BY-4.0"
 "scigrid_gas","","Scientific Grid Model of European Gas Transmission Networks,Gas transmission data model","DLR Institute for Networked Energy Systems","https://web.archive.org/web/20241112092853/https://www.gas.scigrid.de/", "CC-BY-4.0"
 "synthetic_electricity_demand","","Interannual Electricity Demand Calculator,generates country-level electricity consumption time series based on weather data and correlates historical electricity demand to temperature","","https://zenodo.org/records/10820928", "CC-BY-4.0"
+"opsd_electricity_demand","Open Power System Data - Electricity Demand","Time series data of electricity consumption at country-level for various European countries.","Open Power System Data","https://data.open-power-system-data.org/time_series/","unknown"
+"entsoe_electricity_demand","ENTSO-E Transparency Platform - Electricity Demand","Time series data of electricity consumption at country-level for various European countries.","ENTSO-E","https://transparency.entsoe.eu/load/total/dayAhead","unknown"
+"neso_electricity_demand","NESO Electricity Demand Data","Time series data of electricity consumption for Great Britain.","National Energy System Operator (NESO)","https://www.neso.energy/data-portal/historic-demand-data","`NESO Open Data License v1.0 <https://www.neso.energy/data-portal/neso-open-licence>`__"
 "copernicus_land_cover","Copernicus Global Land Service", "Land cover and land use inventory of European continent","Copernicus","https://land.copernicus.eu/en/products/global-dynamic-land-cover", "CC-BY-4.0"
 "ship_raster","Global Shipping Traffic Density","To build ship density raster and use it further to compute availability matrix for renewables","Worldbank","https://datacatalog.worldbank.org/search/dataset/0037580/Global-Shipping-Traffic-Density", "CC-BY-4.0"
 "eez","Maritime Boundaries World EEZ","To estimate potentials for offshore wind in country's EEZ","Marine Regions","https://www.marineregions.org/downloads.php", "CC-BY-4.0"
@@ -41,4 +44,4 @@
 "mobility_profiles","German Vehicle Activity Profiles","Vehicle activity profiles for different vehicle types and road types in Germany, based on monitoring data from the Federal Highway Research Institute (BASt). These profiles provide insights into travel behavior and patterns, which can be used for transport modeling and analysis.","Federal Highway Research Institute (BASt)","https://www.bast.de/DE/Themen/Digitales/HF_1/Massnahmen/verkehrszaehlung/Stundenwerte.html?nn=414410","CC-BY-4.0"
 "dh_areas","","Shapes of district heating areas","ISI Fraunhofer-Institut für System- und Innovationsforschung","https://fordatis.fraunhofer.de/handle/fordatis/341.5","CC-BY-4.0"
 "geothermal_heat_utilisation_potentials","","Potentials for Geothermal heat utilisation","ISI Fraunhofer-Institut für System- und Innovationsforschung","https://fordatis.fraunhofer.de/handle/fordatis/341.5","CC-BY-4.0"
-"jrc_ardeco","Annual Regional Database of the European Commission's Directorate General for Regional and Urban Policy","The database contains a set of long time-series variables and indicators for EU regions, as well as for regions in some EFTA and candidate countries, at various statistical scales (NUTS1, NUTS2, NUTS3, metro regions).","European Commission","https://territorial.ec.europa.eu/ardeco","similar to CC-BY"
+"jrc_ardeco","Annual Regional Database of the European Commission's Directorate General for Regional and Urban Policy","The database contains a set of long time-series variables and indicators for EU regions, as well as for regions in some EFTA and candidate countries, at various statistical scales (NUTS1, NUTS2, NUTS3, metro regions).","European Commission","https://territorial.ec.europa.eu/ardeco","similar to CC-BY"
diff --git a/doc/retrieve.rst b/doc/retrieve.rst
@@ -58,6 +58,51 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
     For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`__.
 
 
+
+Rule ``retrieve_electricity_demand_opsd``
+=========================================
+
+This rule downloads hourly electric load data for each country from the `OPSD platform <https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv>`__.
+
+**Relevant Settings**
+
+None.
+
+**Outputs**
+
+- ``data/electricity_demand_opsd_raw.csv``
+
+Rule ``retrieve_electricity_demand_entsoe``
+===========================================
+
+This rule downloads hourly electric load data for each country from the `ENTSOE Transparency Platform <https://transparency.entsoe.eu>`__.
+
+**Relevant Settings**
+
+None.
+
+**Outputs**
+
+- ``data/electricity_demand_entsoe_raw.csv``
+
+Rule ``retrieve_electricity_demand_neso``
+=========================================
+
+This rule downloads hourly electric load data for the United Kingdom from the `NESO Data Portal <https://www.neso.energy/data-portal/historic-demand-data>`__.
+
+**Relevant Settings**
+
+None.
+
+**Outputs**
+
+- ``data/electricity_demand_neso_raw.csv``
+
+Rule ``retrieve_tyndp_bundle``
+=========================================
+
+.. automodule:: retrieve_tyndp_bundle
+
 Rule ``retrieve_cost_data``
 ================================
 

diff --git a/pixi.toml b/pixi.toml
@@ -93,6 +93,8 @@ xarray = ">=2024.3.0,<2025.7.0"
 xlrd = ">=2.0.2"
 yaml = ">=0.2.5"
 snakemake-storage-plugin-cached-http = ">=0.1.0"
+typer = ">=0.21.0"
+python-dotenv = ">=1.2.1"
 
 [feature.doc.tasks.build-docs]
 args = ["dir", {"arg" = "output", "default" = "html"}]

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
@@ -10,7 +10,9 @@ rule build_electricity_demand:
         countries=config_provider("countries"),
         load=config_provider("load"),
     input:
-        reported=ancient("data/electricity_demand_raw.csv"),
+        opsd=rules.retrieve_electricity_demand_opsd.output["csv"],
+        neso=rules.retrieve_electricity_demand_neso.output["csv"],
+        entsoe=rules.retrieve_electricity_demand_entsoe.output["csv"],
         synthetic=lambda w: (
             ancient(rules.retrieve_synthetic_electricity_demand.output["csv"])
             if config_provider("load", "supplement_synthetic")(w)
@@ -380,8 +382,6 @@ if COUNTRY_RUNOFF_DATASET["source"] == "build":
             logs("build_country_runoff.log"),
         benchmark:
             benchmarks("build_country_runoff")
-        conda:
-            "../envs/environment.yaml"
         script:
             "../scripts/build_country_runoff.py"
 

diff --git a/rules/build_sector.smk b/rules/build_sector.smk
@@ -791,8 +791,6 @@ if (COUNTRY_HDD_DATASET := dataset_version("country_hdd"))["source"] in ["build"
             logs("build_country_hdd.log"),
         benchmark:
             benchmarks("build_country_hdd")
-        conda:
-            "../envs/environment.yaml"
         script:
             "../scripts/build_country_hdd.py"
 
@@ -1266,8 +1264,6 @@ if MOBILITY_PROFILES_DATASET["source"] in ["build"]:
             logs("build_mobility_profiles.log"),
         benchmark:
             benchmarks("build_mobility_profiles")
-        conda:
-            "../envs/environment.yaml"
         script:
             "../scripts/build_mobility_profiles.py"
 

diff --git a/rules/postprocess.smk b/rules/postprocess.smk
@@ -159,8 +159,6 @@ if config["foresight"] != "perfect":
                 RESULTS
                 + "benchmarks/plot_interactive_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}"
             )
-        conda:
-            "../envs/environment.yaml"
         script:
             "../scripts/plot_balance_map_interactive.py"
 

diff --git a/rules/retrieve.smk b/rules/retrieve.smk
@@ -392,18 +392,155 @@ if (SCIGRID_GAS_DATASET := dataset_version("scigrid_gas"))["source"] in [
             unpack_archive(output["zip_file"], output_folder)
 
 
-rule retrieve_electricity_demand:
-    params:
-        versions=["2019-06-05", "2020-10-06"],
-    output:
-        "data/electricity_demand_raw.csv",
-    log:
-        "logs/retrieve_electricity_demand.log",
-    resources:
-        mem_mb=5000,
-    retries: 2
-    script:
-        "../scripts/retrieve_electricity_demand.py"
+if (OPSD_DEMAND_DATA := dataset_version("opsd_electricity_demand"))["source"] in [
+    "primary"
+]:
+
+    rule retrieve_electricity_demand_opsd:
+        params:
+            versions=["2019-06-05", "2020-10-06"],
+        output:
+            csv=f"{OPSD_DEMAND_DATA['folder']}/electricity_demand_opsd_raw.csv",
+        log:
+            "logs/retrieve_electricity_demand_opsd.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        script:
+            "../scripts/retrieve_electricity_demand_opsd.py"
+
+
+if (OPSD_DEMAND_DATA := dataset_version("opsd_electricity_demand"))["source"] in [
+    "archive"
+]:
+
+    rule retrieve_electricity_demand_opsd:
+        input:
+            csv=storage(OPSD_DEMAND_DATA["url"]),
+        output:
+            csv=f"{OPSD_DEMAND_DATA['folder']}/electricity_demand_opsd_raw.csv",
+        retries: 2
+        run:
+            copy2(input["csv"], output["csv"])
+
+
+if (ENTSOE_DEMAND_DATA := dataset_version("entsoe_electricity_demand"))["source"] in [
+    "primary"
+]:
+
+    ENTSOE_COUNTRIES = [
+        "AL",
+        "AT",
+        "BE",
+        "BA",
+        "BG",
+        "CH",
+        "CY",
+        "CZ",
+        "DE",
+        "DK",
+        "EE",
+        "ES",
+        "FI",
+        "FR",
+        "GB",
+        "GR",
+        "HR",
+        "HU",
+        "IE",
+        "IT",
+        "LT",
+        "LU",
+        "LV",
+        "MD",
+        "ME",
+        "MK",
+        "NL",
+        "NO",
+        "PL",
+        "PT",
+        "RO",
+        "RS",
+        "SE",
+        "SI",
+        "SK",
+        "UA",
+        "XK",
+    ]
+
+    rule retrieve_electricity_demand_entsoe_country:
+        params:
+            entsoe_token=config["secrets"]["entsoe_token"],
+        output:
+            csv=f"{ENTSOE_DEMAND_DATA['folder']}"
+            + "/electricity_demand_entsoe_raw_{country}.csv",
+        log:
+            "logs/retrieve_electricity_demand_entsoe_{country}.log",
+        resources:
+            mem_mb=2000,
+        retries: 2
+        script:
+            "../scripts/retrieve_electricity_demand_entsoe.py"
+
+    rule retrieve_electricity_demand_entsoe:
+        input:
+            csvs=expand(
+                f"{ENTSOE_DEMAND_DATA['folder']}"
+                + "/electricity_demand_entsoe_raw_{country}.csv",
+                country=ENTSOE_COUNTRIES,
+            ),
+        output:
+            csv=f"{ENTSOE_DEMAND_DATA['folder']}/electricity_demand_entsoe_raw.csv",
+        run:
+            import pandas as pd
+
+            loads = [pd.read_csv(csv, index_col=0) for csv in input.csvs]
+            df = pd.concat(loads, axis=1, join="outer").sort_index()
+            df.to_csv(output.csv)
+
+
+if (ENTSOE_DEMAND_DATA := dataset_version("entsoe_electricity_demand"))["source"] in [
+    "archive"
+]:
+
+    rule retrieve_electricity_demand_entsoe:
+        input:
+            csv=storage(ENTSOE_DEMAND_DATA["url"]),
+        output:
+            csv=f"{ENTSOE_DEMAND_DATA['folder']}/electricity_demand_entsoe_raw.csv",
+        retries: 2
+        run:
+            copy2(input["csv"], output["csv"])
+
+
+if (NESO_DEMAND_DATA := dataset_version("neso_electricity_demand"))["source"] in [
+    "primary"
+]:
+
+    rule retrieve_electricity_demand_neso:
+        output:
+            csv=f"{NESO_DEMAND_DATA['folder']}/electricity_demand_neso_raw.csv",
+        log:
+            "logs/retrieve_electricity_demand_neso.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        script:
+            "../scripts/retrieve_electricity_demand_neso.py"
+
+
+if (NESO_DEMAND_DATA := dataset_version("neso_electricity_demand"))["source"] in [
+    "archive"
+]:
+
+    rule retrieve_electricity_demand_neso:
+        input:
+            csv=storage(NESO_DEMAND_DATA["url"]),
+        output:
+            csv=f"{NESO_DEMAND_DATA['folder']}/electricity_demand_neso_raw.csv",
+        retries: 2
+        run:
+            copy2(input["csv"], output["csv"])
 
 
 if (
-Original file line number
+Diff line change
@@ Expand Up / @@ -159,8 +159,6 @@ if config["foresight"] != "perfect": @@
                     RESULTS
                     + "benchmarks/plot_interactive_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}"
                 )
-            conda:
-                "../envs/environment.yaml"
             script:
                 "../scripts/plot_balance_map_interactive.py"
@@ Expand Down @@