Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Incremental backup and point in time recovery for XtraBackup #13156

Merged
merged 32 commits into from
Jun 6, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
6212772
incremental backup is always using 'builtin' engine
shlomi-noach May 24, 2023
bc9c029
restore: use 'builtin' for incremental restore
shlomi-noach May 25, 2023
db506ca
test all backup types
shlomi-noach May 25, 2023
1ae6041
format code
shlomi-noach May 25, 2023
1b79d88
Populate PurgedPosition
shlomi-noach May 25, 2023
b17db00
cleanup backups at the end of each test case
shlomi-noach May 25, 2023
7f1a577
improved cleanup
shlomi-noach May 25, 2023
c40090f
rename variable
shlomi-noach May 25, 2023
a042eb6
record all backups
shlomi-noach May 25, 2023
33cb6cd
no need to cleanup backups in between test cases, since each new case…
shlomi-noach May 25, 2023
7b7bd41
install xtrabackup on backup_pitr tests
shlomi-noach May 25, 2023
a8e6817
use pgzip for xtrabackup
shlomi-noach May 28, 2023
a1af0fe
more debug info
shlomi-noach May 28, 2023
6db1a8d
builtin engine: store gtid_purged in manifest
shlomi-noach May 29, 2023
b5641cf
use backupfrom-GTID as incremental-from-GTID if first binary log has …
shlomi-noach May 29, 2023
57877e6
more unit tests
shlomi-noach May 29, 2023
9b9016d
improve error message
shlomi-noach May 29, 2023
a2dc603
capturing MySQL's stderr and reading and logging if not empty
shlomi-noach May 29, 2023
fa66d2c
At the end of Xtrabackup restore, validate that @@gtid_purged (and th…
shlomi-noach May 30, 2023
136126b
add comperssion details into test case. Fix GTID validation of manife…
shlomi-noach May 30, 2023
c28a9ab
check manifest
shlomi-noach May 31, 2023
f40db18
Refactor into function
shlomi-noach May 31, 2023
741cfeb
check manifest.Position.GTIDSet
shlomi-noach May 31, 2023
5fbe3d1
fix wrangler tests
shlomi-noach May 31, 2023
abf6672
typo
shlomi-noach Jun 5, 2023
4f5fd46
Update go/vt/mysqlctl/backup.go
shlomi-noach Jun 5, 2023
289645e
Update go/vt/mysqlctl/backup.go
shlomi-noach Jun 5, 2023
027f33d
Update go/vt/mysqlctl/backup.go
shlomi-noach Jun 5, 2023
2b02ab7
typo
shlomi-noach Jun 5, 2023
fe2aa08
Update go/vt/mysqlctl/mysqld.go
shlomi-noach Jun 5, 2023
63890aa
Update go/vt/mysqlctl/mysqld.go
shlomi-noach Jun 5, 2023
a67e497
Update go/vt/mysqlctl/mysqld.go
shlomi-noach Jun 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
incremental backup is always using 'builtin' engine
Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
  • Loading branch information
shlomi-noach committed May 24, 2023
commit 6212772ed41517fe5ff211969d7e74f36e2cf67b
16 changes: 12 additions & 4 deletions go/vt/mysqlctl/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,16 +135,24 @@ func Backup(ctx context.Context, params BackupParams) error {
return vterrors.Wrap(err, "StartBackup failed")
}

be, err := GetBackupEngine()
if err != nil {
return vterrors.Wrap(err, "failed to find backup engine")
}
// Scope stats to selected backup engine.
beParams := params.Copy()
beParams.Stats = params.Stats.Scope(
stats.Component(stats.BackupEngine),
stats.Implementation(titleCase(backupEngineImplementation)),
)
var be BackupEngine
if isIncrementalBackup(beParams) {
// Incremental backups are always done via 'builtin' engine, which copies
// appropriate binlog files.
be = BackupRestoreEngineMap[builtinBackupEngineName]
} else {
be, err = GetBackupEngine()
if err != nil {
return vterrors.Wrap(err, "failed to find backup engine")
}
}

// Take the backup, and either AbortBackup or EndBackup.
usable, err := be.ExecuteBackup(ctx, beParams, bh)
logger := params.Logger
Expand Down
5 changes: 5 additions & 0 deletions go/vt/mysqlctl/backupengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,11 @@ func init() {
}
}

// isIncrementalBackup is a convenience function to check whether the params indicate an incremental backup request
func isIncrementalBackup(params BackupParams) bool {
return params.IncrementalFromPos != ""
}

func registerBackupEngineFlags(fs *pflag.FlagSet) {
fs.StringVar(&backupEngineImplementation, "backup_engine_implementation", backupEngineImplementation, "Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup.")
}
Expand Down
7 changes: 1 addition & 6 deletions go/vt/mysqlctl/builtinbackupengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,6 @@ func registerBuiltinBackupEngineFlags(fs *pflag.FlagSet) {
fs.UintVar(&builtinBackupFileWriteBufferSize, "builtinbackup-file-write-buffer-size", builtinBackupFileWriteBufferSize, "write files using an IO buffer of this many bytes. Golang defaults are used when set to 0.")
}

// isIncrementalBackup is a convenience function to check whether the params indicate an incremental backup request
func isIncrementalBackup(params BackupParams) bool {
return params.IncrementalFromPos != ""
}

// fullPath returns the full path of the entry, based on its type
func (fe *FileEntry) fullPath(cnf *Mycnf) (string, error) {
// find the root to use
Expand Down Expand Up @@ -1126,5 +1121,5 @@ func getPrimaryPosition(ctx context.Context, tmc tmclient.TabletManagerClient, t
}

func init() {
BackupRestoreEngineMap["builtin"] = &BuiltinBackupEngine{}
BackupRestoreEngineMap[builtinBackupEngineName] = &BuiltinBackupEngine{}
}
13 changes: 11 additions & 2 deletions go/vt/mysqlctl/xtrabackupengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,18 @@ func closeFile(wc io.WriteCloser, fileName string, logger logutil.Logger, finalE
}
}

// ExecuteBackup returns a boolean that indicates if the backup is usable,
// ExecuteBackup runs a backup based on given params. This could be a full or incremental backup.
// The function returns a boolean that indicates if the backup is usable, and an overall error.
func (be *XtrabackupEngine) ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (bool, error) {
params.Logger.Infof("Executing Backup at %v for keyspace/shard %v/%v on tablet %v, concurrency: %v, compress: %v, incrementalFromPos: %v",
params.BackupTime, params.Keyspace, params.Shard, params.TabletAlias, params.Concurrency, backupStorageCompress, params.IncrementalFromPos)

return be.executeFullBackup(ctx, params, bh)
}

// executeFullBackup returns a boolean that indicates if the backup is usable,
// and an overall error.
func (be *XtrabackupEngine) ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (complete bool, finalErr error) {
func (be *XtrabackupEngine) executeFullBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (complete bool, finalErr error) {

if params.IncrementalFromPos != "" {
return false, vterrors.New(vtrpc.Code_INVALID_ARGUMENT, "incremental backups not supported in xtrabackup engine.")
Expand Down