feat(backup): route a database's backups to its own target (#1085)

arc.toml

@@ -372,8 +372,45 @@ local_path = "./data/backups"           # Directory for local backups; see the r
 #
 # A named backup TARGET makes the destination configurable, and it may be remote
 # (#1085). Point default_target at it and local_path above is ignored entirely —
-# the directory is not even created. Exactly ONE target is supported in this
-# release; per-database routing across several comes later.
+# the directory is not even created. SEVERAL targets are supported, with
+# per-database routing: `databases = [...]` on a target sends those databases
+# files there and everything else goes to default_target.
+#
+# A routed backup writes one manifest and one file sidecar PER TARGET, each
+# describing its own slice and each naming the whole run, plus an index at
+# <backup_id>/index.json on the default target that names every target the run
+# touched. The listing then returns ONE entry per backup id, with every
+# destination holding a slice of it; a target that will not answer is named in
+# unreachable_targets and the rest of the listing still answers. A delete
+# sweeps every target. A restore reads EVERY target of the run and REFUSES
+# before writing anything if one of them is not configured here, will not
+# answer, or holds no manifest for the id — restoring only the reachable part
+# would report success over a set it did not restore.
+#
+# A database goes to exactly one target: naming it on two is refused at
+# startup. Naming databases on default_target is allowed and does nothing,
+# since everything unrouted goes there already, so put them on the target you
+# want them to go TO: in the example below, default_target is "main" and only
+# [backup.targets.audit] carries `databases`.
+#
+# Only the COMMA separates names. A database name may contain a space, so
+# databases = "my db" is ONE name and databases = ["a", "b"] is two.
+#
+# A target nothing is routed to is INERT: no backup writes to it, and a warning
+# at startup names it. It is not a refusal, because adding the target block and
+# its databases in separate commits is ordinary, and it is deliberately not a
+# leg either — a leg is probed before anything is copied, so a target nothing
+# routes to would otherwise make an unrelated store a precondition of every
+# backup in this instance, including whole-instance ones. The names are storage-root
+# segments, exactly as a scoped backup names them, so an edge-sync spoke is
+# named as the SPOKE: ["prod"] does not mean spoke1/prod.
+#
+# The instance-wide state always goes to default_target whatever the routing
+# says, because it belongs to no one database: the SQLite database, the Iceberg
+# SQL catalog, the Iceberg table metadata, an out-of-root Iceberg warehouse and
+# this config file. A database's own field schema anchors (_schema/<db>/) and
+# compaction recovery state (_compaction_state/<tier>/<db>/) DO travel with its
+# data to its target.
 #
 # Target names are letters, digits and underscore only, and are lowercased. The
 # rule that matters is the HYPHEN: it cannot appear in an environment variable
@@ -384,8 +421,10 @@ local_path = "./data/backups"           # Directory for local backups; see the r
 # and "audit" are one target.
 #
 # Arc REFUSES AT STARTUP a destination that overlaps primary storage or the
-# tiered-storage cold tier, for two reasons that are both permanent and both
-# silent. A destination inside the storage root is inventoried by the next
+# tiered-storage cold tier, or ANOTHER backup target, for two reasons that are
+# both permanent and both silent (two targets that contain one another are one
+# listing, so each backup leg would enumerate the other objects and deleting
+# one backup id would reach both). A destination inside the storage root is inventoried by the next
 # backup, so each backup copies the previous one in full; on a cluster those
 # files are absent from the Raft manifest, and past the skip ratio every
 # replace-mode restore is refused. And with reconciliation enabled and
@@ -399,10 +438,18 @@ local_path = "./data/backups"           # Directory for local backups; see the r
 # "s3.us-east-1.amazonaws.com" on the other does not make them two places. One
 # bucket with DISJOINT prefixes is fine and is the intended shape.
 #
-# default_target = "audit"
+# default_target = "main"
+#
+# [backup.targets.main]                  # the DEFAULT: everything unrouted goes here
+# type = "local"                         # local, s3, minio, azure or azblob
+# local_path = "/srv/arc-backups"
 #
-# [backup.targets.audit]
-# type = "s3"                            # local, s3, minio, azure or azblob
+# [backup.targets.audit]                 # a ROUTED target: it carries `databases`
+# type = "s3"
+# databases = ["audit", "compliance"]    # these databases go here; everything else to default_target
+#                                        # also "audit,compliance" or
+#                                        # ARC_BACKUP_TARGETS_AUDIT_DATABASES=audit,compliance
+#                                        # the COMMA is the only separator: "my db" is ONE name
 # s3_bucket = "acme-arc-audit-backups"
 # s3_prefix = "arc/backups"
 # s3_region = "us-east-1"
@@ -414,15 +461,20 @@ local_path = "./data/backups"           # Directory for local backups; see the r
 #
 # From the environment ALONE, with nothing in this file, a target takes an extra
 # variable, because a name cannot be discovered from a map that does not exist:
-#   ARC_BACKUP_TARGET_NAMES=audit
+#   ARC_BACKUP_TARGET_NAMES=main,audit
+#   ARC_BACKUP_TARGETS_MAIN_TYPE=s3
+#   ARC_BACKUP_TARGETS_MAIN_S3_BUCKET=acme-arc-backups
 #   ARC_BACKUP_TARGETS_AUDIT_TYPE=s3
 #   ARC_BACKUP_TARGETS_AUDIT_S3_BUCKET=acme-arc-audit-backups
-#   ARC_BACKUP_DEFAULT_TARGET=audit
+#   ARC_BACKUP_TARGETS_AUDIT_DATABASES=audit,compliance
+#   ARC_BACKUP_DEFAULT_TARGET=main
 #
-# With a REMOTE target, include_config defaults to FALSE for every backup: this
-# file carries that target own credentials, so the default would put the keys
-# that unlock the backup store inside the backups it holds. A request may ask
-# for it explicitly and is warned.
+# With ANY remote target configured, include_config defaults to FALSE for every
+# backup — not only when the DEFAULT target is remote: this file carries every
+# target's own credentials, so a local default plus one remote routed target
+# still means the default would put the keys that unlock that store inside a
+# backup it holds. A request may ask for it explicitly and is warned, naming
+# every remote target whose keys the backup then carries.
 #
 # Backups carry the identity of the instance that wrote them, so two instances
 # sharing one bucket and prefix do not merge listings. On a cluster that

cmd/arc/main.go

@@ -4356,30 +4356,35 @@ func main() {
 					Msg("iceberg.warehouse is not a local path; backups cannot include its table metadata")
 			}
 		}
-		// Backup destination (#1085 stage B2b-1). A configured target replaces
-		// the local directory entirely; with none, BackupPath is the
-		// destination exactly as before. config.Load has already validated the
-		// target, bounded its key prefix, and refused an overlap with primary
-		// storage or the cold tier — see
+		// Backup destinations (#1085 stage B2b-1, a set with per-database
+		// routing since B2b-2). A configured target replaces the local
+		// directory entirely; with none, BackupPath is the destination exactly
+		// as before. config.Load has already validated every target, bounded
+		// each key prefix, refused the same database on two targets, and
+		// refused an overlap with primary storage, with the cold tier and
+		// between any two targets — see
 		// config.checkBackupDestinationOverlap, which records what that
 		// refusal actually protects against: a backup that re-copies itself
 		// every run, and the reconciliation sweep deleting the backups.
-		var backupTarget *backup.Target
-		if t := cfg.Backup.DefaultBackupTarget(); t != nil {
+		var backupTargets []backup.Target
+		for _, name := range cfg.Backup.TargetNamesSorted() {
+			t := cfg.Backup.Targets[name]
 			keyPrefix, err := t.KeyPrefix()
 			if err != nil {
 				// Unreachable: validateBackupTargets asks for the same prefix
 				// at load and refuses a bad one. Fatal rather than silently
 				// reserving no key headroom, which would turn an overlong key
-				// from a reported skip into a failed write.
+				// from a reported skip into a failed write. Per target, so a
+				// routed target with an unusable prefix is as loud as the
+				// default one.
 				log.Fatal().Err(err).Str("target", t.Name).Msg("Backup target prefix is unusable")
 			}
-			backupTarget = &backup.Target{
+			backupTargets = append(backupTargets, backup.Target{
 				Name:      t.Name,
 				Spec:      t.BackendSpec(),
 				KeyPrefix: keyPrefix,
 				Remote:    t.IsRemote(),
-			}
+			})
 		}
 
 		// Backup owner identity (#1085 stage B2b-1). The CLUSTER when
@@ -4416,10 +4421,18 @@ func main() {
 			backupInstanceID = id
 		}
 
+		// Built once: the ready log below reports its size, and rebuilding it
+		// there would walk every target a second time to answer a question
+		// this map already answers.
+		backupRouting := cfg.Backup.RoutingMap()
+
 		backupManager, err := backup.NewManager(&backup.ManagerConfig{
 			DataStorage:            storageBackend,
 			BackupPath:             cfg.Backup.LocalPath,
-			Target:                 backupTarget,
+			Targets:                backupTargets,
+			DefaultTarget:          cfg.Backup.DefaultTarget,
+			Routing:                backupRouting,
+			OperationTimeout:       cfg.Backup.OperationTimeout,
 			InstanceID:             backupInstanceID,
 			SQLiteDBPath:           cfg.Auth.DBPath,
 			IcebergCatalogDBPath:   icebergCatalogDBPath,
@@ -4472,11 +4485,15 @@ func main() {
 			// backup goes somewhere it does not, and that directory is not
 			// even created in that configuration.
 			ready := log.Info()
-			if backupTarget != nil {
+			if len(backupTargets) > 0 {
+				names := make([]string, 0, len(backupTargets))
+				for _, t := range backupTargets {
+					names = append(names, t.Name)
+				}
 				ready = ready.
-					Str("backup_target", backupTarget.Name).
-					Str("backup_target_type", backupTarget.Spec.Type).
-					Bool("backup_target_remote", backupTarget.Remote)
+					Str("backup_default_target", cfg.Backup.DefaultTarget).
+					Strs("backup_targets", names).
+					Int("routed_databases", len(backupRouting))
 			} else {
 				ready = ready.Str("backup_path", cfg.Backup.LocalPath)
 			}