Allow subdirectories for add & delete granules

examples/granules.yaml

@@ -9,6 +9,9 @@ store: <name of the layer>
 
 # Internal path to directory within Geoserver where images are placed
 geoserver_target_dir: /mnt/images/this/layers/own/directory
+# Whether to keep the subpath given with the filename in program arguments
+# This should be true if the images are placed in subdirectories of the target directory
+keep_subpath: false
 # For S3, use the https address of the bucket:
 # geoserver_target_dir: https://<bucket name>.<host address>
 
@@ -49,12 +52,11 @@ layers:
   airmass: satellite_geo_europe_seviri-15min_airmass
   ash: satellite_geo_europe_seviri-15min_ash
 
-
 log_config:
   version: 1
   formatters:
     fmt:
-      format: '[%(asctime)s %(levelname)-8s %(name)s] %(message)s'
+      format: "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
   handlers:
     console:
       class: logging.StreamHandler
@@ -70,15 +72,15 @@ log_config:
       interval: 1
       backupCount: 10
   loggers:
-    '':
+    "":
       level: DEBUG
       handlers: [console, file]
       propagate: false
-    'georest':
+    "georest":
       level: DEBUG
       handlers: [console, file]
       propagate: false
-    'georest.utils':
+    "georest.utils":
       level: DEBUG
       handlers: [console, file]
       propagate: false

georest/__init__.py

@@ -286,7 +286,7 @@
     This function wraps some boilerplate around adding a granule to a layer.
 
     """
-    fname = utils.convert_file_path(config, fname_in)
+    fname = utils.convert_file_path(config, fname_in, keep_subpath=config.get("keep_subpath", False))
     identity_check_seconds = config.get("identity_check_seconds")
 
     store = _get_store_name_from_filename(config, fname)
@@ -397,15 +397,14 @@
     fname = os.path.basename(granule["properties"]["location"])
     logger.debug("Removing granule '%s' from %s:%s", fname, workspace, store)
     cat.delete_granule(store, store_obj, id_, workspace)
-    logger.info("Granule '%s' removed from %s:%s",
-                fname, workspace, store)
+    logger.info("Granule '%s' removed from %s:%s", fname, workspace, store)
 
 
 def _delete_files_from_fs(config, gs_location):
     delete_files = config.get("delete_files", False)
     if not delete_files:
         return
-    fs_path = utils.convert_file_path(config, gs_location, inverse=True)
+    fs_path = utils.convert_file_path(config, gs_location, inverse=True, keep_subpath=config.get("keep_subpath", False))
     _delete_file_from_fs(config, fs_path)
     _delete_file_from_fs(config, fs_path, replace_extension="prj")
 

georest/tests/test_geoserver.py

@@ -246,10 +246,11 @@ def test_get_layer_coverage():
     "passwd": "passwd",
     "workspace": "satellite",
     "geoserver_target_dir": "/mnt/data",
+    "keep_subpath": False,
     "file_pattern": "{area}_{productname}.tif",
     "layer_id": "productname",
     "layers": {"airmass": "airmass_store"},
-    }
+}
 
 
 @mock.patch("georest.utils.file_in_granules")

georest/tests/test_utils.py

@@ -323,3 +323,44 @@ def test_run_posttroll_adder_s3(connect_to_gs_catalog, add_s3_granule,
         config,
         expected_meta
     )
+
+
+def test_convert_file_path():
+    """Test the file path conversion for the default case."""
+    from georest.utils import convert_file_path
+
+    config = {"geoserver_target_dir": "/geoserver/internal/path/"}
+    res = convert_file_path(config, "/external/path/file.tif")
+
+    assert res == "/geoserver/internal/path/file.tif"
+
+
+def test_convert_file_path_inverse():
+    """Test the file path conversion for the inverse case."""
+    from georest.utils import convert_file_path
+
+    config = {"exposed_base_dir": "/external/path/"}
+    res = convert_file_path(config, "/geoserver/internal/path/file.tif", inverse=True)
+
+    assert res == "/external/path/file.tif"
+
+
+def test_convert_file_path_keep_subpath():
+    """Test the file path conversion when keeping the subpath of the file."""
+    from georest.utils import convert_file_path
+
+    config = {"geoserver_target_dir": "/geoserver/internal/path/"}
+    res = convert_file_path(config, "subpath/file.tif", keep_subpath=True)
+
+    assert res == "/geoserver/internal/path/subpath/file.tif"
+
+
+def test_convert_file_path_keep_subpath_inverse():
+    """Test the file path conversion when keeping the subpath of the file for the inverse case."""
+    from georest.utils import convert_file_path
+
+    config = {"exposed_base_dir": "/external/path/",
+              "geoserver_target_dir": "/geoserver/internal/path/"}
+    res = convert_file_path(config, "/geoserver/internal/path/subpath/file.tif", inverse=True, keep_subpath=True)
+
+    assert res == "/external/path/subpath/file.tif"

georest/utils.py

@@ -85,15 +85,21 @@ def _write_property_zip(prop_paths, zip_path):
     return zip_path
 
 
-def convert_file_path(config, file_path, inverse=False):
+def convert_file_path(config, file_path, inverse=False, keep_subpath=False):
     """Convert given file path to internal directory structure."""
-    basename = os.path.basename(file_path)
+    if not keep_subpath:
+        basename = os.path.basename(file_path)
+    else:
+        basename = file_path
     if inverse:
         new_dir = config["exposed_base_dir"]
     else:
         new_dir = config["geoserver_target_dir"]
 
-    path = os.path.join(new_dir, basename)
+    if keep_subpath and inverse:
+        path = basename.replace(config["geoserver_target_dir"], config["exposed_base_dir"])
+    else:
+        path = os.path.join(new_dir, basename)
 
     return path