Added gv_at data source; Restructured download and merging part

manage.py

@@ -12,57 +12,47 @@
 
 
 @app.cli.command()
-def download():
+def prepare():
     """
     Downloads SRTM tiles to disk. Can be specified over minx, maxx, miny, maxy.
-
-    :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max
-        in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp
-    :type xyrange: comma-separated integers
     """
 
     filestreams.download()
     log.info("Downloaded all files")
+
+
+@app.cli.command()
+def merge():
+
+    filestreams.merge_data()
+    log.info("Merged downloaded files")
 
 
 @app.cli.command()
 def create():
     """Creates all tables defined in models.py"""
 
     db.create_all()
-    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm']))
-    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite']))
+    for table in SETTINGS['provider_parameters']['tables']:
+        log.info("Table {} was created.".format(table))
 
 
 @app.cli.command()
 def drop():
     """Drops all tables defined in models.py"""
 
     db.drop_all()
-    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_srtm']))
-    log.info("Table {} was created.".format(SETTINGS['provider_parameters']['table_name_composite']))
+    for table in SETTINGS['provider_parameters']['tables']:
+        log.info("Table {} was dropped.".format(table))
+
 
 @app.cli.command()
 def importdata(): 
     """
     Imports all data found in ./tiles
-
-    :param xyrange: A comma-separated list of x_min, x_max, y_min, y_max
-        in that order. For reference grid, see http://srtm.csi.cgiar.org/SELECTION/inputCoord.asp
-    :type xyrange: comma-separated integers
     """
     log.info("Starting to import data...")
 
     filestreams.raster2pgsql()
 
     log.info("Imported data successfully!")
-
-
-def _arg_format(xy_range_txt):
-
-    str_split = [int(s.strip()) for s in xy_range_txt.split(',')]
-
-    xy_range = [[str_split[0], str_split[2]],
-                [str_split[1], str_split[3]]]
-
-    return xy_range

openelevationservice/server/__init__.py

@@ -33,14 +33,15 @@ def create_app(script_info=None):
 
     # set up extensions
     db.init_app(app)
-    
+
     provider_details = SETTINGS['provider_parameters']
-    log.info("Following provider parameters are active:\n"
-              "Host:\t{host}\n"
-              "DB:\t{db_name}\n"
-              "Table1:\t{table_name_terrestrial}\n"
-              "Table2:\t{table_name_bathymetry}\n"
-              "User:\t{user_name}".format(**provider_details))
+
+    for table in provider_details['tables']:
+        log.info("Following provider parameters are active:\n"
+                  "Host:\t{host}\n"
+                  "DB:\t{db_name}\n"
+                  "Table:\t{table_name}\n"
+                  "User:\t{user_name}".format(**provider_details, table_name=table))
 
     # register blueprints
     from openelevationservice.server.api.views import main_blueprint

openelevationservice/server/config.py

@@ -2,6 +2,7 @@
 
 from openelevationservice import SETTINGS
 
+
 class BaseConfig(object):
     """Base configuration."""
 

openelevationservice/server/db_import/filestreams.py

@@ -3,26 +3,92 @@
 from openelevationservice import TILES_DIR, SETTINGS
 from openelevationservice.server.sources import PROVIDER_MAPPING
 from openelevationservice.server.utils.logger import get_logger
-# from openelevationservice.server.db_import import raster_processing
 
 from os import path, environ
 import subprocess
 
 log = get_logger(__name__)
 
+# TODO: write Exception
+
 
 def download():
-    """
-    Downlaods GMTED and SRTM v4.1 tiles as bytestream and saves them to TILES_DIR.
-    """
+    """Selects download provider."""
+
+    extent_settings = SETTINGS['tables']['terrestrial']['extent']
+    download_list = []
+
+    if not SETTINGS['provider_parameters']['tables']:
+        log.error("Please define at least one table in ops_settings.yml")
+
+    else:
+        # for table in SETTINGS['provider_parameters']['tables']:
+        if 'terrestrial' in SETTINGS['provider_parameters']['tables']:
+
+            # only SRTM data download
+            if 60 >= extent_settings['max_y'] >= -60:
+                provider = PROVIDER_MAPPING['terrestrial']['srtm']()
+                download_list.append(provider)
+
+            # SRTM and GMTED data download
+            elif extent_settings['max_y'] > 60 >= extent_settings['min_y'] or \
+                    extent_settings['max_y'] > -60 >= extent_settings['min_y']:
+                for source in SETTINGS['tables']['terrestrial']['sources']:
+                    provider = PROVIDER_MAPPING['terrestrial'][source]()
+                    download_list.append(provider)
+
+            # only GMTED data download
+            else:
+                provider = PROVIDER_MAPPING['terrestrial']['gmted']()
+                download_list.append(provider)
+
+        else:
+            for p in PROVIDER_MAPPING:
+                if p != 'terrestrial':
+                    provider = PROVIDER_MAPPING[p]()
+                    download_list.append(provider)
+
+        for download_provider in download_list:
+            try:
+                download_provider.download_data()
+            except Exception as err:
+                log.error(err)
 
-    for table in SETTINGS['tables']:
-        for source in SETTINGS['tables'][table]['sources']:
-            provider = PROVIDER_MAPPING[source]()
+
+def merge_data():
+    """Selects provider for further raster tile resampling and merging."""
+
+    extent_settings = SETTINGS['tables']['terrestrial']['extent']
+    merge_list = []
+
+    if not SETTINGS['provider_parameters']['tables']:
+        log.error("Please define at least one table in ops_settings.yml")
+
+    else:
+        for table in SETTINGS['provider_parameters']['tables']:
+            if 'terrestrial' in table:
+
+                # only SRTM data
+                if 60 >= extent_settings['max_y'] >= -60:
+                    provider = PROVIDER_MAPPING['terrestrial']['srtm']()
+                    merge_list.append(provider)
+
+                # GMTED data or GMTED and SRTM data
+                elif extent_settings['max_y'] > 60 or extent_settings['min_y'] < -60:
+                    provider = PROVIDER_MAPPING['terrestrial']['gmted']()
+                    merge_list.append(provider)
+
+            else:
+                for p in PROVIDER_MAPPING:
+                    if p != 'terrestrial':
+                        provider = PROVIDER_MAPPING[p]()
+                        merge_list.append(provider)
+
+        for merge_provider in merge_list:
             try:
-                provider.download_data()
+                merge_provider.merge_tiles()
             except Exception as err:
-                log.info(err)
+                log.error(err)
 
 
 def raster2pgsql():
@@ -32,49 +98,36 @@ def raster2pgsql():
     :raises subprocess.CalledProcessError: Raised when raster2pgsql throws an error.
     """
 
+    # TODO: define -t spatial size
+    # TODO: bathy_raster.tif -> just one row?
+    # TODO: import single files, too?
     pg_settings = SETTINGS['provider_parameters']
 
     # Copy all env variables and add PGPASSWORD
     env_current = environ.copy()
     env_current['PGPASSWORD'] = pg_settings['password']
 
-    tiles_dir_name = None
-
-    if SETTINGS["sources"][0]["type"] == "cgiar_csi":
+    for table in SETTINGS['provider_parameters']['tables']:
+        filename = path.join(TILES_DIR + '/' + table + '_raster.tif')
 
         # Tried to import every raster individually by user-specified xyrange
         # similar to download(), but raster2pgsql fuck it up somehow.. The PostGIS
         # raster will not be what one would expect. Instead doing a bulk import of all files.
-        cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_srtm} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}"
+        cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -t 50x50 {filename} {table_name} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}"
         # -s: raster SRID
         # -a: append to table (assumes it's been create with 'create()')
         # -C: apply all raster Constraints
-        # -P: pad tiles to guarantee all tiles have the same width and height
         # -M: vacuum analyze after import
         # -t: specifies the pixel size of each row. Important to keep low for performance!
 
-        tiles_dir_name = TILES_DIR
-
-    else:
-        cmd_raster2pgsql = r"raster2pgsql -s 4326 -a -C -M -P -t 50x50 {filename} {table_name_joerd} | psql -q -h {host} -p {port} -U {user_name} -d {db_name}"
+        cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': filename, 'table_name': table, **pg_settings})
 
-        tiles_dir_name = TILES_DIR
+        proc = subprocess.Popen(cmd_raster2pgsql,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT,
+                                shell=True,
+                                env=env_current)
 
-    if path.join(TILES_DIR, '*.tif'):
-        cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.tif'), **pg_settings})
-    else:
-        cmd_raster2pgsql = cmd_raster2pgsql.format(**{'filename': path.join(tiles_dir_name, '*.img'), **pg_settings})
-
-    proc = subprocess.Popen(cmd_raster2pgsql,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.STDOUT,
-                            shell=True,
-                            env=env_current
-                            )
-
-#    for line in proc.stdout:
-#        log.debug(line.decode())
-#    proc.stdout.close()
-    return_code = proc.wait()
-    if return_code:
-        raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql)
+        return_code = proc.wait()
+        if return_code:
+            raise subprocess.CalledProcessError(return_code, cmd_raster2pgsql)

openelevationservice/server/db_import/models.py

@@ -4,35 +4,22 @@
 from openelevationservice.server.utils import logger
 
 from flask_sqlalchemy import SQLAlchemy
-from sqlalchemy import Index
 from geoalchemy2 import Raster
 
 db = SQLAlchemy()
 
 log = logger.get_logger(__name__)
-table_name_terrestrial = SETTINGS['provider_parameters']['table_name_terrestrial']
-table_name_bathymetry = SETTINGS['provider_parameters']['table_name_bathymetry']
 
 
 class Cgiar(db.Model):
     """Database model for SRTM v4.1 aka CGIAR dataset."""
 
-    __tablename__ = table_name_terrestrial
+    for table in SETTINGS['provider_parameters']['tables']:
 
-    rid = db.Column(db.Integer, primary_key=True)
-    rast = db.Column(Raster)
+        __tablename__ = table
 
-    def __repr__(self):
-        return '<rid {}, rast {}>'.format(self.rid, self.rast)
+        rid = db.Column(db.Integer, primary_key=True)
+        rast = db.Column(Raster)
 
-
-class Joerd(db.Model):
-    """Database model for SRTM v4.1 aka CGIAR dataset."""
-
-    __tablename__ = table_name_bathymetry
-
-    rid = db.Column(db.Integer, primary_key=True)
-    rast = db.Column(Raster)
-
-    def __repr__(self):
-        return '<rid {}, rast {}>'.format(self.rid, self.rast)
+        def __repr__(self):
+            return '<rid {}, rast {}>'.format(self.rid, self.rast)