Skip to content

Commit c3db8af

Browse files
SlachSlach
committed
- fix --restore-database-mapping behavior for ATTACH MATERIALIZED VIEW corner cases, fix Altinity#559
Signed-off-by: Slach <bloodjazman@gmail.com>
1 parent 5114e90 commit c3db8af

File tree

5 files changed

+51
-34
lines changed

5 files changed

+51
-34
lines changed

‎ChangeLog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ BUG FIXES
1515
- `create` and `restore` commands will respect `skip_tables` config options and `--table` cli parameter, to avoid create unnecessary empty databases, fix [583](https://github.com/AlexAkulov/clickhouse-backup/issues/583)
1616
- fix `watch` unexpected connection closed behavior, fix [568](https://github.com/AlexAkulov/clickhouse-backup/issues/568)
1717
- fix `watch` validation parameters corner cases, close [569](https://github.com/AlexAkulov/clickhouse-backup/pull/569)
18+
- fix `--restore-database-mapping` behavior for `ATTACH MATERIALIZED VIEW` corner cases, fix [559](https://github.com/AlexAkulov/clickhouse-backup/issues/559)
1819

1920
# v2.1.2
2021
IMPROVEMENTS

‎pkg/backup/restore.go

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,15 @@ func (b *Backuper) restoreDataEmbedded(backupName string, tablesForRestore ListO
523523
}
524524

525525
func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, tablePattern string, tablesForRestore ListOfTables, diskMap map[string]string, disks []clickhouse.Disk, log *apexLog.Entry) error {
526+
if len(b.cfg.General.RestoreDatabaseMapping) > 0 {
527+
for _, targetDb := range b.cfg.General.RestoreDatabaseMapping {
528+
if tablePattern != "" {
529+
tablePattern += "," + targetDb + ".*"
530+
} else {
531+
tablePattern += targetDb + ".*"
532+
}
533+
}
534+
}
526535
chTables, err := b.ch.GetTables(ctx, tablePattern)
527536
if err != nil {
528537
return err
@@ -550,21 +559,16 @@ func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, ta
550559
}
551560
}
552561
dstTablesMap := map[metadata.TableTitle]clickhouse.Table{}
553-
for i := range chTables {
562+
for i, chTable := range chTables {
554563
dstTablesMap[metadata.TableTitle{
555564
Database: chTables[i].Database,
556565
Table: chTables[i].Name,
557-
}] = chTables[i]
566+
}] = chTable
558567
}
559568

560569
var missingTables []string
561570
for _, tableForRestore := range tablesForRestore {
562571
found := false
563-
if len(b.cfg.General.RestoreDatabaseMapping) > 0 {
564-
if targetDB, isMapped := b.cfg.General.RestoreDatabaseMapping[tableForRestore.Database]; isMapped {
565-
tableForRestore.Database = targetDB
566-
}
567-
}
568572
for _, chTable := range chTables {
569573
if (tableForRestore.Database == chTable.Database) && (tableForRestore.Table == chTable.Name) {
570574
found = true

‎pkg/backup/table_pattern.go

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,12 @@ func getTableListByPatternLocal(cfg *config.Config, metadataPath string, tablePa
146146
return result, nil
147147
}
148148

149-
var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.([^\s\x60.]*)(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.))?`)
149+
var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.([^\s\x60.]*)(?:( UUID '[^']+'))?(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.))?(?:(.+FROM )(\x60?)([^\s\x60.]*)(\x60?)(\.))?`)
150150
var createRE = regexp.MustCompile(`(?m)^CREATE`)
151151
var attachRE = regexp.MustCompile(`(?m)^ATTACH`)
152152
var uuidRE = regexp.MustCompile(`UUID '[a-f\d\-]+'`)
153153

154-
var replicatedRE = regexp.MustCompile(`(Replicated[a-zA-Z]*MergeTree)\('([^']+)'([^\)]+)\)`)
154+
var replicatedRE = regexp.MustCompile(`(Replicated[a-zA-Z]*MergeTree)\('([^']+)'([^)]+)\)`)
155155
var distributedRE = regexp.MustCompile(`(Distributed)\(([^,]+),([^,]+),([^)]+)\)`)
156156

157157
func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRule map[string]string) error {
@@ -161,12 +161,24 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu
161161
// substitute database in the table create query
162162
var substitution string
163163

164-
if len(createRE.FindAllString(originTable.Query, -1)) > 0 {
164+
if createRE.MatchString(originTable.Query) {
165165
// matching CREATE... command
166166
substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}", targetDB)
167-
} else if len(attachRE.FindAllString(originTable.Query, -1)) > 0 {
168-
// matching ATTACH...TO... command
169-
substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}%v${11}", targetDB, targetDB)
167+
} else if attachRE.MatchString(originTable.Query) {
168+
matches := queryRE.FindAllStringSubmatch(originTable.Query, -1)
169+
if matches[0][4] != originTable.Database {
170+
return fmt.Errorf("invalid SQL: %s for restore-database-mapping[%s]=%s", originTable.Query, originTable.Database, targetDB)
171+
}
172+
setMatchedDb := func(clauseTargetDb string) string {
173+
if clauseMappedDb, isClauseMapped := dbMapRule[clauseTargetDb]; isClauseMapped {
174+
clauseTargetDb = clauseMappedDb
175+
}
176+
return clauseTargetDb
177+
}
178+
toClauseTargetDb := setMatchedDb(matches[0][10])
179+
fromClauseTargetDb := setMatchedDb(matches[0][15])
180+
// matching ATTACH ... TO .. SELECT ... FROM ... command
181+
substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}%v${11}${12}${13}${14}%v${16}${17}", targetDB, toClauseTargetDb, fromClauseTargetDb)
170182
} else {
171183
if originTable.Query == "" {
172184
continue

‎test/integration/install_delve.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ CGO_ENABLED=0 GO111MODULE=on go install -ldflags "-s -w -extldflags '-static'" g
2424
# /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- server
2525
# /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- restore --schema test_backup_8007633179464680930
2626
# CLICKHOUSE_TIMEOUT=3m CLICKHOUSE_DEBUG=true LOG_LEVEL=debug /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- watch --watch-interval=1m --full-interval=2m
27-
27+
# LOG_LEVEL=debug /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- restore --data --restore-database-mapping database1:database2 --tables database1.* test_restore_database_mapping
2828
# run integration_test.go under debug, run from host OS not inside docker
2929
# go test -timeout 30m -failfast -tags=integration -run "TestIntegrationEmbedded" -v ./test/integration/integration_test.go -c -o ./test/integration/integration_test
3030
# sudo -H bash -c 'export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server; export COMPOSE_FILE=docker-compose_advanced.yml; export CLICKHOUSE_VERSION=head; cd ./test/integration/; /root/go/bin/dlv --listen=127.0.0.1:40002 --headless=true --api-version=2 --accept-multiclient exec ./integration_test -- -test.timeout 30m -test.failfast -test.run "TestIntegrationEmbedded"'

‎test/integration/integration_test.go

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -693,34 +693,34 @@ func TestRestoreDatabaseMapping(t *testing.T) {
693693
ch.queryWithNoError(r, "CREATE DATABASE database1")
694694
ch.queryWithNoError(r, "CREATE TABLE database1.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t1','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
695695
ch.queryWithNoError(r, "CREATE TABLE database1.d1 AS database1.t1 ENGINE=Distributed('{cluster}',database1, t1)")
696+
ch.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
697+
ch.queryWithNoError(r, "CREATE MATERIALIZED VIEW database1.mv1 TO database1.t2 AS SELECT * FROM database1.t1")
696698
ch.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2022-01-01 00:00:00', number FROM numbers(10)")
697699

698700
log.Info("Create backup")
699701
r.NoError(dockerExec("clickhouse", "clickhouse-backup", "create", testBackupName))
700-
r.NoError(dockerExec("clickhouse", "clickhouse-backup", "restore", "--restore-database-mapping", "database1:database2", testBackupName))
702+
r.NoError(dockerExec("clickhouse", "clickhouse-backup", "restore", "--schema", "--rm", "--restore-database-mapping", "database1:database2", "--tables", "database1.*", testBackupName))
703+
r.NoError(dockerExec("clickhouse", "clickhouse-backup", "restore", "--data", "--restore-database-mapping", "database1:database2", "--tables", "database1.*", testBackupName))
701704

702705
ch.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)")
703706

704707
log.Info("Check result")
705-
result := make([]int, 0)
706-
r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database2.t1"))
707-
r.Equal(1, len(result), "expect one row")
708-
r.Equal(10, result[0], "expect count=10")
709-
710-
result = make([]int, 0)
711-
r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database2.d1"))
712-
r.Equal(1, len(result), "expect one row")
713-
r.Equal(10, result[0], "expect count=10")
714-
715-
result = make([]int, 0)
716-
r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database1.t1"))
717-
r.Equal(1, len(result), "expect one row")
718-
r.Equal(20, result[0], "expect count=20")
719-
720-
result = make([]int, 0)
721-
r.NoError(ch.chbackend.Select(&result, "SELECT count() FROM database1.d1"))
722-
r.Equal(1, len(result), "expect one row")
723-
r.Equal(20, result[0], "expect count=20")
708+
checkRecordset := func(expectedRows, expectedCount int, query string) {
709+
result := make([]int, 0)
710+
r.NoError(ch.chbackend.Select(&result, query))
711+
r.Equal(expectedRows, len(result), "expect %d row", expectedRows)
712+
r.Equal(expectedCount, result[0], "expect count=%d", expectedCount)
713+
}
714+
715+
// database 2
716+
checkRecordset(1, 10, "SELECT count() FROM database2.t1")
717+
checkRecordset(1, 10, "SELECT count() FROM database2.d1")
718+
checkRecordset(1, 10, "SELECT count() FROM database2.mv1")
719+
720+
// database 1
721+
checkRecordset(1, 20, "SELECT count() FROM database1.t1")
722+
checkRecordset(1, 20, "SELECT count() FROM database1.d1")
723+
checkRecordset(1, 20, "SELECT count() FROM database1.mv1")
724724

725725
fullCleanup(r, ch, []string{testBackupName}, []string{"local"}, databaseList, true)
726726
}

0 commit comments

Comments
 (0)