upload with sftp and fixes on upload S3

Upload to a remote host using SFTP with support for password and public
key authentication. Host keys are checked but it can be disabled.

Fix the purge of remote files that affected S3 upload as well.

README.md

@@ -159,13 +159,22 @@ dumping) are used as shell globs to choose which files to decrypt.
 ### Upload to remote locations
 
 All files produced by a run can be uploaded to a remote location by setting the
-`--upload` option to a value different than `none`. The possible values are `s3` or
-`none`.
+`--upload` option to a value different than `none`. The possible values are
+`s3`, `sftp` or `none`.
 
 When set to `s3`, files are uploaded to AWS S3. The `--s3-*` family of options
 can be used to tweak the access to the bucket. The `--s3-profile` option only reads
 credentials and basic configuration, s3 specific options are not used.
 
+When set to `sftp`, files are uploaded to a remote host using SFTP. The
+`--sftp-*` family of options can be used to setup the access to the host. The
+`PGBK_SSH_PASS` sets the password or decrypts the private key (identity file),
+it is used only when `--sftp-password` is not set (either in the configuration
+file or on the command line). When an identity file is provided, the password
+is used to decrypt it and the password authentication method is not tried with
+the server. The only SSH authentication methods used are password and
+publickey. If an SSH agent is available, it is always used.
+
 The `--purge-remote` option can be set to `yes` to apply the same purge policy
 on the remote location as the local directory.
 

config.go

@@ -76,7 +76,7 @@ type options struct {
 	CipherPassphrase string
 	Decrypt          bool
 
-	Upload       string // values are none, s3
+	Upload       string // values are none, s3, sftp
 	PurgeRemote  bool
 	S3Region     string
 	S3Bucket     string
@@ -86,6 +86,14 @@ type options struct {
 	S3Secret     string
 	S3ForcePath  bool
 	S3DisableTLS bool
+
+	SFTPHost             string
+	SFTPPort             string
+	SFTPUsername         string
+	SFTPPassword         string
+	SFTPDirectory        string
+	SFTPIdentityFile     string // path to private key
+	SFTPIgnoreKnownHosts bool
 }
 
 func defaultOptions() options {
@@ -250,6 +258,14 @@ func parseCli(args []string) (options, []string, error) {
 	S3ForcePath := pflag.String("s3-force-path", "no", "force path style addressing instead of virtual hosted bucket\naddressing")
 	S3UseTLS := pflag.String("s3-tls", "yes", "enable or disable TLS on requests")
 
+	pflag.StringVar(&opts.SFTPHost, "sftp-host", "", "Remote hostname for SFTP")
+	pflag.StringVar(&opts.SFTPPort, "sftp-port", "", "Remote port for SFTP")
+	pflag.StringVar(&opts.SFTPUsername, "sftp-user", "", "Login for SFTP when different than the current user")
+	pflag.StringVar(&opts.SFTPPassword, "sftp-password", "", "Password for SFTP or passphrase when identity file is set")
+	pflag.StringVar(&opts.SFTPDirectory, "sftp-directory", "", "Target directory on the remote host")
+	pflag.StringVar(&opts.SFTPIdentityFile, "sftp-identity", "", "Path to a private key")
+	SFTPIgnoreHostKey := pflag.String("sftp-ignore-hostkey", "no", "Check the target host key against local known hosts")
+
 	pflag.StringVarP(&opts.Host, "host", "h", "", "database server host or socket directory")
 	pflag.IntVarP(&opts.Port, "port", "p", 0, "database server port number")
 	pflag.StringVarP(&opts.Username, "username", "U", "", "connect as specified database user")
@@ -377,7 +393,7 @@ func parseCli(args []string) (options, []string, error) {
 	}
 
 	// Validate upload option
-	stores := []string{"none", "s3"}
+	stores := []string{"none", "s3", "sftp"}
 	if err := validateEnum(opts.Upload, stores); err != nil {
 		return opts, changed, fmt.Errorf("invalid value for --upload: %s", err)
 	}
@@ -387,20 +403,25 @@ func parseCli(args []string) (options, []string, error) {
 		return opts, changed, fmt.Errorf("invalid value for --purge-remote: %s", err)
 	}
 
-	// Validate S3 options
-	opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath)
-	if err != nil {
-		return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err)
-	}
+	switch opts.Upload {
+	case "s3":
+		// Validate S3 options
+		opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath)
+		if err != nil {
+			return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err)
+		}
 
-	S3WithTLS, err := validateYesNoOption(*S3UseTLS)
-	if err != nil {
-		return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err)
-	}
-	opts.S3DisableTLS = !S3WithTLS
+		S3WithTLS, err := validateYesNoOption(*S3UseTLS)
+		if err != nil {
+			return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err)
+		}
+		opts.S3DisableTLS = !S3WithTLS
 
-	if opts.Upload == "s3" && opts.S3Bucket == "" {
-		return opts, changed, fmt.Errorf("option --s3-bucket is mandatory when --upload=s3")
+	case "sftp":
+		opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey)
+		if err != nil {
+			return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err)
+		}
 	}
 
 	return opts, changed, nil
@@ -457,6 +478,14 @@ func loadConfigurationFile(path string) (options, error) {
 	opts.S3ForcePath = s.Key("s3_force_path").MustBool(false)
 	opts.S3DisableTLS = !s.Key("s3_tls").MustBool(true)
 
+	opts.SFTPHost = s.Key("sftp_host").MustString("")
+	opts.SFTPPort = s.Key("sftp_port").MustString("")
+	opts.SFTPUsername = s.Key("sftp_user").MustString("")
+	opts.SFTPPassword = s.Key("sftp_password").MustString("")
+	opts.SFTPDirectory = s.Key("sftp_directory").MustString("")
+	opts.SFTPIdentityFile = s.Key("sftp_identity").MustString("")
+	opts.SFTPIgnoreKnownHosts = s.Key("sftp_ignore_hostkey").MustBool(false)
+
 	// Validate purge keep and time limit
 	keep, err := validatePurgeKeepValue(purgeKeep)
 	if err != nil {
@@ -488,15 +517,11 @@ func loadConfigurationFile(path string) (options, error) {
 	}
 
 	// Validate upload option
-	stores := []string{"none", "s3"}
+	stores := []string{"none", "s3", "sftp"}
 	if err := validateEnum(opts.Upload, stores); err != nil {
 		return opts, fmt.Errorf("invalid value for upload: %s", err)
 	}
 
-	if opts.Upload == "s3" && opts.S3Bucket == "" {
-		return opts, fmt.Errorf("option s3_bucket is mandatory when upload is s3")
-	}
-
 	// Validate the value of the timestamp format. Force the use of legacy
 	// on windows to avoid failure when creating filenames with the
 	// timestamp
@@ -688,6 +713,21 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin
 		case "s3-tls":
 			opts.S3DisableTLS = cliOpts.S3DisableTLS
 
+		case "sftp-host":
+			opts.SFTPHost = cliOpts.SFTPHost
+		case "sftp-port":
+			opts.SFTPPort = cliOpts.SFTPPort
+		case "sftp-user":
+			opts.SFTPUsername = cliOpts.SFTPUsername
+		case "sftp-password":
+			opts.SFTPPassword = cliOpts.SFTPPassword
+		case "sftp-directory":
+			opts.SFTPDirectory = cliOpts.SFTPDirectory
+		case "sftp-identity":
+			opts.SFTPIdentityFile = cliOpts.SFTPIdentityFile
+		case "sftp-ignore-hostkey":
+			opts.SFTPIgnoreKnownHosts = cliOpts.SFTPIgnoreKnownHosts
+
 		case "host":
 			opts.Host = cliOpts.Host
 		case "port":

config_test.go

@@ -375,7 +375,7 @@ func TestParseCli(t *testing.T) {
 				},
 				false,
 				false,
-				"invalid value for --upload: value not found in [none s3]",
+				"invalid value for --upload: value not found in [none s3 sftp]",
 				"",
 			},
 		}

go.mod

@@ -10,6 +10,8 @@ require (
 	github.com/jackc/pgconn v1.8.1 // indirect
 	github.com/jackc/pgtype v1.7.0
 	github.com/jackc/pgx/v4 v4.11.0
+	github.com/pkg/sftp v1.13.4 // indirect
 	github.com/spf13/pflag v1.0.5
+	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e // indirect
 	gopkg.in/ini.v1 v1.62.0
 )

go.sum

@@ -198,6 +198,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -270,6 +272,8 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pkg/sftp v1.13.4 h1:Lb0RYJCmgUcBgZosfoi9Y9sbl6+LJgOIgk/2Y4YjMFg=
+github.com/pkg/sftp v1.13.4/go.mod h1:LzqnAvaD5TWeNBsZpfKxSYn1MbjWwOsCIAFFJbpIsK8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -328,6 +332,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -361,8 +366,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
+golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -389,6 +397,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -419,6 +428,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg=
 golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -489,6 +499,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

main.go

@@ -193,6 +193,10 @@ func run() (retVal error) {
 		return fmt.Errorf("provided pg_dump is older than 8.4, unable use it.")
 	}
 
+	if opts.Upload == "s3" && opts.S3Bucket == "" {
+		return fmt.Errorf("a bucket is mandatory when upload is s3")
+	}
+
 	// Parse the connection information
 	l.Verboseln("processing input connection parameters")
 	conninfo, err := prepareConnInfo(opts.Host, opts.Port, opts.Username, opts.ConnDb)
@@ -428,6 +432,11 @@ func run() (retVal error) {
 		if err != nil {
 			return fmt.Errorf("failed to prepare upload to S3: %w", err)
 		}
+	case "sftp":
+		repo, err = NewSFTPRepo(opts)
+		if err != nil {
+			return fmt.Errorf("failed to prepare upload over SFTP: %w", err)
+		}
 	}
 
 	for _, dbname := range databases {
@@ -442,7 +451,7 @@ func run() (retVal error) {
 		}
 
 		if opts.PurgeRemote {
-			if err := purgeRemoteDumps(repo, dbname, o.PurgeKeep, limit); err != nil {
+			if err := purgeRemoteDumps(repo, opts.Directory, dbname, o.PurgeKeep, limit); err != nil {
 				retVal = err
 			}
 		}
@@ -455,7 +464,7 @@ func run() (retVal error) {
 		}
 
 		if opts.PurgeRemote {
-			if err := purgeRemoteDumps(repo, other, defDbOpts.PurgeKeep, limit); err != nil {
+			if err := purgeRemoteDumps(repo, opts.Directory, other, defDbOpts.PurgeKeep, limit); err != nil {
 				retVal = err
 			}
 		}
@@ -654,6 +663,7 @@ func relPath(basedir, path string) string {
 	for strings.HasPrefix(target, "../") {
 		target = strings.TrimPrefix(target, "../")
 	}
+
 	return target
 }
 
@@ -1201,6 +1211,14 @@ func postProcessFiles(inFiles chan sumFileJob, wg *sync.WaitGroup, opts options)
 		if err != nil {
 			l.Errorln("failed to prepare upload to S3:", err)
 			ret <- err
+			repo = nil
+		}
+	case "sftp":
+		repo, err = NewSFTPRepo(opts)
+		if err != nil {
+			l.Errorln("failed to prepare upload over SFTP:", err)
+			ret <- err
+			repo = nil
 		}
 	}
 
@@ -1213,6 +1231,7 @@ func postProcessFiles(inFiles chan sumFileJob, wg *sync.WaitGroup, opts options)
 				j, more := <-uploadIn
 				if !more {
 					wg.Done()
+					done <- true
 					l.Verboseln("stopped upload worker", id)
 					return
 				}
@@ -1252,6 +1271,15 @@ func postProcessFiles(inFiles chan sumFileJob, wg *sync.WaitGroup, opts options)
 			<-done
 		}
 		close(uploadIn)
+
+		for i := 0; i < opts.Jobs; i++ {
+			<-done
+		}
+
+		if repo != nil {
+			repo.Close()
+		}
+
 	}()
 
 	return ret

pg_back.conf

@@ -114,6 +114,21 @@ upload = none
 # s3_force_path = false
 # s3_tls = true
 
+# SFTP Access information. If the user is empty, the current system user is
+# used. Port defaults to 22. The password is also used as passphrase for any
+# identity file given, it can be provided with the PGBK_SSH_PASS environment
+# variable. PGBK_SSH_PASS is overridden by a value set here or on the command
+# line. Use the directory to inform where to store files, it can be relative to
+# the working directory of the SSH connection, the home directory of the remote
+# user in most cases.
+# sftp_host =
+# sftp_port =
+# sftp_user =
+# sftp_password =
+# sftp_directory =
+# sftp_identity =
+# sftp_ignore_hostkey = false
+
 # # Per database options. Use a ini section named the same as the
 # # database. These options take precedence over the global values
 # [dbname]