From d5508ed3cae46f9004f103e82368a90e34dd7b61 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 08:52:14 +0200 Subject: [PATCH 1/8] pep8 --- src/egon/data/processing/osmtgmod/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/egon/data/processing/osmtgmod/__init__.py b/src/egon/data/processing/osmtgmod/__init__.py index 2586181cb..30645a38c 100644 --- a/src/egon/data/processing/osmtgmod/__init__.py +++ b/src/egon/data/processing/osmtgmod/__init__.py @@ -53,7 +53,7 @@ def import_osm_data(): ) else: - subproc.run( + subproc.run( [ "git", "clone", @@ -160,7 +160,6 @@ def import_osm_data(): ) logging.info("Importing OSM-Data...") - # After updating OSM-Data, power_tables (for editing) # have to be updated as well logging.info("Creating power-tables...") From 7bc2c383fe732fb0a40e2dfb1dd0cb45c6ead5c1 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 09:06:47 +0200 Subject: [PATCH 2/8] set CWD as java's temp dir when executing osmosis --- src/egon/data/processing/osmtgmod/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/egon/data/processing/osmtgmod/__init__.py b/src/egon/data/processing/osmtgmod/__init__.py index 30645a38c..03b1d5c1d 100644 --- a/src/egon/data/processing/osmtgmod/__init__.py +++ b/src/egon/data/processing/osmtgmod/__init__.py @@ -6,6 +6,7 @@ import datetime import logging import codecs +from pathlib import Path import egon.data.config from egon.data.config import settings import egon.data.subprocess as subproc @@ -142,9 +143,10 @@ def import_osm_data(): ) subproc.run( - "%s --read-pbf %s --write-pgsql \ + "JAVACMD_OPTIONS='%s' %s --read-pbf %s --write-pgsql \ database=%s host=%s user=%s password=%s" % ( + f"-Djava.io.tmpdir={Path('.')}", os.path.join(egon.data.__path__[0], "processing/osmtgmod/osmTGmod/", config["osm_data"]["osmosis_path_to_binary"]), From 178accf320e43abc7ae4fbc8ab6cec4ca4b180a3 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 09:07:48 +0200 Subject: [PATCH 3/8] add notes to readme --- README.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 506e9bb32..4d9f7848f 100644 --- a/README.rst +++ b/README.rst @@ -122,9 +122,12 @@ packages are required too. Right now these are: * To download ERA5 weather data you need to register at the CDS registration page and install the CDS API key as described `here `_ - You also have to agree on the `terms of use + You also have to agree on the `terms of use `_ +* Make sure you have enough free disk space (~300 GB) in your working + directory. + Installation ============ @@ -188,6 +191,10 @@ solution. can't be run on laptop. Use the :ref:`test mode ` for experimenting. +.. warning:: + + A complete run of the workflow needs loads of free disk space (~300 GB) to + store (temporary) files. Test mode --------- @@ -206,4 +213,4 @@ Data is reduced during execution of the workflow to represent only this area. Further Reading =============== -You can find more in depth documentation at https://eGon-data.readthedocs.io. +You can find more in-depth documentation at https://eGon-data.readthedocs.io. From becd95ab764a3e40d56a947cd6a9d671a8a5a6a5 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 09:08:15 +0200 Subject: [PATCH 4/8] amend authors --- AUTHORS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 5685c1427..0d7e707a0 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -2,4 +2,4 @@ Authors ======= -* Guido Pleßmann, Ilka Cußman, Stephan Günther - https://github.com/openego/eGon-data +* Guido Pleßmann, Ilka Cußman, Stephan Günther, Jonathan Amme - https://github.com/openego/eGon-data From 6b97c333f7a63ba0babf36547ba24c677c1984a3 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 09:11:19 +0200 Subject: [PATCH 5/8] amend changelog --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 045e5d006..1bdba07e1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -192,3 +192,5 @@ Bug fixes `#258 `_ * Fix conflicting docker containers by setting a project name `#289 `_ +* Set current working directory as java's temp dir when executing osmosis + `#344 `_ From 769137ebf5ea9a2c655b3d30d574b6bf860ef9ad Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 09:22:08 +0200 Subject: [PATCH 6/8] use subdir of cwd --- src/egon/data/processing/osmtgmod/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/egon/data/processing/osmtgmod/__init__.py b/src/egon/data/processing/osmtgmod/__init__.py index 03b1d5c1d..693aa6601 100644 --- a/src/egon/data/processing/osmtgmod/__init__.py +++ b/src/egon/data/processing/osmtgmod/__init__.py @@ -142,11 +142,16 @@ def import_osm_data(): {config['osm_data']['osmosis_path_to_binary']}""" ) + # create directory to store osmosis' temp files + osmosis_temp_dir = Path('.') / "osmosis_temp/" + if not os.path.exists(osmosis_temp_dir): + os.mkdir(osmosis_temp_dir) + subproc.run( "JAVACMD_OPTIONS='%s' %s --read-pbf %s --write-pgsql \ database=%s host=%s user=%s password=%s" % ( - f"-Djava.io.tmpdir={Path('.')}", + f"-Djava.io.tmpdir={osmosis_temp_dir}", os.path.join(egon.data.__path__[0], "processing/osmtgmod/osmTGmod/", config["osm_data"]["osmosis_path_to_binary"]), From d3340c8f1e970b466fb77fa222ac251669ea5e00 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 09:54:23 +0200 Subject: [PATCH 7/8] fix path joins to make more filesafe --- src/egon/data/processing/osmtgmod/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/egon/data/processing/osmtgmod/__init__.py b/src/egon/data/processing/osmtgmod/__init__.py index 693aa6601..6d17d3514 100644 --- a/src/egon/data/processing/osmtgmod/__init__.py +++ b/src/egon/data/processing/osmtgmod/__init__.py @@ -26,8 +26,7 @@ def run_osmtgmod(): target_path = osm_config["target"]["path_testmode"] filtered_osm_pbf_path_to_file = os.path.join( - egon.data.__path__[0] + "/datasets" + "/osm/" - + target_path + egon.data.__path__[0], "datasets", "osm", target_path ) docker_db_config = db.credentials() @@ -75,8 +74,7 @@ def import_osm_data(): target_path = osm_config["target"]["path_testmode"] filtered_osm_pbf_path_to_file = os.path.join( - egon.data.__path__[0] + "/datasets" + "/osm/" - + target_path + egon.data.__path__[0], "datasets", "osm", target_path ) docker_db_config=db.credentials() @@ -153,8 +151,8 @@ def import_osm_data(): % ( f"-Djava.io.tmpdir={osmosis_temp_dir}", os.path.join(egon.data.__path__[0], - "processing/osmtgmod/osmTGmod/", - config["osm_data"]["osmosis_path_to_binary"]), + "processing/osmtgmod/osmTGmod/", + config["osm_data"]["osmosis_path_to_binary"]), filtered_osm_pbf_path_to_file, config_database, config["postgres_server"]["host"] From 2f213a4054734b983c828223618b5a309e6e35fe Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 22 Jul 2021 12:09:08 +0200 Subject: [PATCH 8/8] update note on reqd. disk space --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 4d9f7848f..645e52f07 100644 --- a/README.rst +++ b/README.rst @@ -125,7 +125,7 @@ packages are required too. Right now these are: You also have to agree on the `terms of use `_ -* Make sure you have enough free disk space (~300 GB) in your working +* Make sure you have enough free disk space (~350 GB) in your working directory. Installation @@ -193,7 +193,7 @@ solution. .. warning:: - A complete run of the workflow needs loads of free disk space (~300 GB) to + A complete run of the workflow needs loads of free disk space (~350 GB) to store (temporary) files. Test mode