Add the --upload-prefix option to put files under this remote directory

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## pg_back 2.3.2
+
+* Add the --upload-prefix option to put files under this remote directory
+
 ## pg_back 2.3.1
 
 * Fix cipher_public_key and cipher_private_key not allowed in config file

README.md

@@ -207,6 +207,10 @@ the configuration file).
 WARNING: Azure support is not guaranted because there are no free solutions for
 testing on it
 
+The `--upload-prefix` option can be used to place the files in a remote
+directory, as most cloud storage treat prefix as directories. The filename and
+the prefix is separated by a / in the remote location.
+
 The `--purge-remote` option can be set to `yes` to apply the same purge policy
 on the remote location as the local directory.
 

config.go

@@ -83,6 +83,7 @@ type options struct {
 	DumpOnly          bool
 
 	Upload       string // values are none, s3, sftp, gcs
+	UploadPrefix string
 	Download     string // values are none, s3, sftp, gcs
 	ListRemote   string // values are none, s3, sftp, gcs
 	PurgeRemote  bool
@@ -288,6 +289,7 @@ func parseCli(args []string) (options, []string, error) {
 	pflag.StringVar(&opts.CipherPrivateKey, "cipher-private-key", "", "AGE private key for decryption; in Bech32 encoding starting with 'AGE-SECRET-KEY-1'\n")
 
 	pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload")
+	pflag.StringVar(&opts.UploadPrefix, "upload-prefix", "", "add this prefix to uploaded files, similar to a target directory")
 	pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files")
 	pflag.StringVar(&opts.ListRemote, "list-remote", "none", "list the remote files on s3, gcs, sftp, azure instead of dumping. DBNAMEs become\nglobs to select files")
 	purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory")
@@ -502,7 +504,7 @@ func validateConfigurationFile(cfg *ini.File) error {
 		"sftp_port", "sftp_user", "sftp_password", "sftp_directory", "sftp_identity",
 		"sftp_ignore_hostkey", "gcs_bucket", "gcs_endpoint", "gcs_keyfile",
 		"azure_container", "azure_account", "azure_key", "azure_endpoint", "pg_dump_options",
-		"dump_role_passwords", "dump_only",
+		"dump_role_passwords", "dump_only", "upload_prefix",
 	}
 
 gkLoop:
@@ -597,6 +599,7 @@ func loadConfigurationFile(path string) (options, error) {
 	opts.EncryptKeepSrc = s.Key("encrypt_keep_source").MustBool(false)
 
 	opts.Upload = s.Key("upload").MustString("none")
+	opts.UploadPrefix = s.Key("upload_prefix").MustString("")
 	opts.PurgeRemote = s.Key("purge_remote").MustBool(false)
 
 	opts.S3Region = s.Key("s3_region").MustString("")
@@ -832,6 +835,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin
 
 		case "upload":
 			opts.Upload = cliOpts.Upload
+		case "upload-prefix":
+			opts.UploadPrefix = cliOpts.UploadPrefix
 		case "download":
 			opts.Download = cliOpts.Download
 		case "list-remote":

main.go

@@ -542,7 +542,7 @@ func run() (retVal error) {
 		}
 
 		if opts.PurgeRemote && repo != nil {
-			if err := purgeRemoteDumps(repo, opts.Directory, dbname, o.PurgeKeep, limit); err != nil {
+			if err := purgeRemoteDumps(repo, opts.UploadPrefix, opts.Directory, dbname, o.PurgeKeep, limit); err != nil {
 				retVal = err
 			}
 		}
@@ -555,7 +555,7 @@ func run() (retVal error) {
 		}
 
 		if opts.PurgeRemote && repo != nil {
-			if err := purgeRemoteDumps(repo, opts.Directory, other, defDbOpts.PurgeKeep, limit); err != nil {
+			if err := purgeRemoteDumps(repo, opts.UploadPrefix, opts.Directory, other, defDbOpts.PurgeKeep, limit); err != nil {
 				retVal = err
 			}
 		}
@@ -1518,7 +1518,8 @@ func postProcessFiles(inFiles chan sumFileJob, wg *sync.WaitGroup, opts options)
 				}
 
 				if opts.Upload != "none" && repo != nil {
-					if err := repo.Upload(j.Path, relPath(opts.Directory, j.Path)); err != nil {
+					// Prepend the global prefix to the relative path of the dump
+					if err := repo.Upload(j.Path, filepath.Join(opts.UploadPrefix, relPath(opts.Directory, j.Path))); err != nil {
 						l.Errorln(err)
 						if !failed {
 							ret <- err

purge.go

@@ -204,15 +204,17 @@ func purgeDumps(directory string, dbname string, keep int, limit time.Time) erro
 	return nil
 }
 
-func purgeRemoteDumps(repo Repo, directory string, dbname string, keep int, limit time.Time) error {
+func purgeRemoteDumps(repo Repo, uploadPrefix string, directory string, dbname string, keep int, limit time.Time) error {
 	l.Verboseln("remote purge:", dbname, "limit:", limit, "keep:", keep)
 
 	// The dbname can be put in the directory tree of the dump, in this
 	// case the directory containing {dbname} in its name is kept on the
 	// remote path along with any subdirectory. So we have to include it in
 	// the filter when listing remote files
 	dirpath := filepath.Dir(formatDumpPath(directory, "", "", dbname, time.Time{}, 0))
-	prefix := relPath(directory, filepath.Join(dirpath, cleanDBName(dbname)))
+	prefix := filepath.Join(uploadPrefix, relPath(directory, filepath.Join(dirpath, cleanDBName(dbname))))
+
+	l.Verboseln("remote file prefix:", prefix)
 
 	// Get the list of files from the repository, this includes the
 	// contents of dumps in the directory format.