Add --upload.file_regex parameter

conf/mongodb-consistent-backup.example.conf

@@ -68,6 +68,7 @@ production:
   #    key: [zabbix item key]
   upload:
     method: none
+  #  file_regex: [regular expression]  (default: none)
   #  remove_uploaded: [true|false] (default: false)
   #  retries: [1+]           (default: 5)
   #  threads: [1+]           (default: 4)

mongodb_consistent_backup/Upload/S3/S3.py

@@ -24,6 +24,10 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs):
         self.s3_acl              = self.config.upload.s3.acl
         self.key_prefix          = base_dir
         self.validate_bucket     = not self.config.upload.s3.skip_bucket_validation
+        if self.config.upload.file_regex == "none":
+            self.upload_file_regex = None
+        else:
+            self.upload_file_regex = self.config.upload.file_regex
 
         self.threads(self.config.upload.threads)
         self._pool = None
@@ -65,7 +69,7 @@ def run(self):
                 self.chunk_size_mb,
                 self.retries
             ))
-            for file_path in get_upload_files(self.backup_dir):
+            for file_path in get_upload_files(self.backup_dir, self.upload_file_regex):
                 key_name = self.get_key_name(file_path)
                 self._pool.upload(file_path, key_name)
             self._pool.wait()

mongodb_consistent_backup/Upload/S3/S3UploadPool.py

@@ -38,6 +38,7 @@ def __init__(self, bucket_name, region, access_key, secret_key, threads=4, remov
         self.chunk_bytes     = chunk_bytes
         self.key_acl         = key_acl
         self.validate_bucket = kwargs.get("validate_bucket")
+        self.upload_file_regex = kwargs.get("upload_file_regex")
 
         self.multipart_min_bytes = 5242880
 

mongodb_consistent_backup/Upload/Util.py

@@ -1,9 +1,25 @@
 import os
+import re
+import logging
 
 
-def get_upload_files(backup_dir):
+def get_upload_files(backup_dir, regex=None):
     upload_files = []
+    r = None
+    if regex is not None:
+        logging.info("Only uploading files from %s that match the specified regex" % backup_dir)
+        r = re.compile(regex)
+    else:
+        logging.debug("No regex specified, uploading all files in %s" % backup_dir)
+
     for root, dirs, files in os.walk(backup_dir):
-        for file in files:
-            upload_files.append(os.path.join(root, file))
+        for f in files:
+            if r is not None:
+                if r.search(f):
+                    logging.info("Adding %s to files to upload, as it matches the regex" % f)
+                    upload_files.append(os.path.join(root, f))
+                else:
+                    logging.debug("Skipping %s because it does not match the regex" % f)
+            else:
+                upload_files.append(os.path.join(root, f))
     return upload_files

mongodb_consistent_backup/Upload/__init__.py

@@ -10,4 +10,6 @@ def config(parser):
                         help="Number of times to retry upload attempts (default: 5)")
     parser.add_argument("--upload.threads", dest="upload.threads", default=4, type=int,
                         help="Number of threads to use for upload (default: 4)")
+    parser.add_argument("--upload.file_regex", dest="upload.file_regex", default='none', type=str,
+                        help="Limit uploaded file names to those matching a regular expression. (default: none)")
     return parser