cherry pick pingcap#33500 to release-5.2

Signed-off-by: joccau <zak.zhao@pingcap.com>

br/pkg/glue/glue.go

@@ -36,6 +36,7 @@ type Session interface {
 	CreateDatabase(ctx context.Context, schema *model.DBInfo) error
 	CreateTable(ctx context.Context, dbName model.CIStr, table *model.TableInfo) error
 	Close()
+	GetGlobalVariable(name string) (string, error)
 }
 
 // Progress is an interface recording the current execution progress.

br/pkg/gluetidb/glue.go

@@ -146,6 +146,11 @@ func (gs *tidbSession) Close() {
 	gs.se.Close()
 }
 
+// GetGlobalVariables implements glue.Session.
+func (gs *tidbSession) GetGlobalVariable(name string) (string, error) {
+	return gs.se.GetSessionVars().GlobalVarsAccessor.GetTiDBTableValue(name)
+}
+
 // showCreateTable shows the result of SHOW CREATE TABLE from a TableInfo.
 func (gs *tidbSession) showCreateTable(tbl *model.TableInfo) (string, error) {
 	table := tbl.Clone()

br/pkg/task/backup.go

@@ -250,6 +250,16 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
 		statsHandle = mgr.GetDomain().StatsHandle()
 	}
 
+	se, err := g.CreateSession(mgr.GetStorage())
+	if err != nil {
+		return errors.Trace(err)
+	}
+	newCollationEnable, err := se.GetGlobalVariable(tidbNewCollationEnabled)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	log.Info("get newCollationEnable for check during restore", zap.String("newCollationEnable", newCollationEnable))
+
 	client, err := backup.NewBackupClient(ctx, mgr)
 	if err != nil {
 		return errors.Trace(err)
@@ -342,6 +352,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
 			m.ClusterId = req.ClusterId
 			m.ClusterVersion = clusterVersion
 			m.BrVersion = brVersion
+			m.NewCollationsEnabled = newCollationEnable
 		})
 		pdAddress := strings.Join(cfg.PD, ",")
 		log.Warn("Nothing to backup, maybe connected to cluster for restoring",
@@ -438,6 +449,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
 		m.ClusterId = req.ClusterId
 		m.ClusterVersion = clusterVersion
 		m.BrVersion = brVersion
+		m.NewCollationsEnabled = newCollationEnable
 	})
 
 	skipChecksum := !cfg.Checksum || isIncrementalBackup

br/pkg/task/common.go

@@ -73,7 +73,8 @@ const (
 	defaultGRPCKeepaliveTime    = 10 * time.Second
 	defaultGRPCKeepaliveTimeout = 3 * time.Second
 
-	unlimited = 0
+	unlimited               = 0
+	tidbNewCollationEnabled = "new_collation_enabled"
 )
 
 // TLSConfig is the common configuration for TLS connection.

br/pkg/version/version.go

@@ -97,6 +97,7 @@ func CheckClusterVersion(ctx context.Context, client pd.Client, checker VerCheck
 			if err := checkTiFlashVersion(s); err != nil {
 				return errors.Trace(err)
 			}
+			continue
 		}
 
 		tikvVersionString := removeVAndHash(s.Version)

br/tests/br_check_new_collocation_enable/config/new_collation_enable_false.toml

@@ -0,0 +1,16 @@
+# config of tidb
+
+# Schema lease duration
+# There are lot of ddl in the tests, setting this
+# to 360s to test whether BR is gracefully shutdown.
+lease = "360s"
+
+new_collations_enabled_on_first_bootstrap = false
+
+[security]
+ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
+cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key"

br/tests/br_check_new_collocation_enable/config/new_collation_enable_true.toml

@@ -0,0 +1,16 @@
+# config of tidb
+
+# Schema lease duration
+# There are lot of ddl in the tests, setting this
+# to 360s to test whether BR is gracefully shutdown.
+lease = "360s"
+
+new_collations_enabled_on_first_bootstrap = true
+
+[security]
+ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
+cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
+cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
+cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key"

br/tests/br_check_new_collocation_enable/run.sh

@@ -0,0 +1,102 @@
+#!/bin/sh
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source $cur/../_utils/run_services
+
+PROGRESS_FILE="$TEST_DIR/progress_unit_file"
+rm -rf $PROGRESS_FILE
+
+run_sql "CREATE DATABASE $DB;"
+
+run_sql "CREATE TABLE $DB.usertable1 ( \
+  YCSB_KEY varchar(64) NOT NULL, \
+  FIELD0 varchar(1) DEFAULT NULL, \
+  PRIMARY KEY (YCSB_KEY) \
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+
+run_sql "INSERT INTO $DB.usertable1 VALUES (\"a\", \"b\");"
+run_sql "INSERT INTO $DB.usertable1 VALUES (\"aa\", \"b\");"
+
+run_sql "CREATE TABLE $DB.usertable2 ( \
+  YCSB_KEY varchar(64) NOT NULL, \
+  FIELD0 varchar(1) DEFAULT NULL, \
+  PRIMARY KEY (YCSB_KEY) \
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+
+run_sql "INSERT INTO $DB.usertable2 VALUES (\"c\", \"d\");"
+
+# backup db
+echo "backup start ... with brv4.0.8 without NewCollactionEnable"
+bin/brv4.0.8 backup db --db "$DB" -s "local://$TEST_DIR/$DB" \
+    --ca "$TEST_DIR/certs/ca.pem" \
+    --cert "$TEST_DIR/certs/br.pem" \
+    --key "$TEST_DIR/certs/br.key" \
+    --pd $PD_ADDR \
+    --check-requirements=false
+
+# restore db from v4.0.8 version without `newCollationEnable`
+echo "restore start ... without NewCollactionEnable in backupmeta"
+restore_fail=0
+error_str="NewCollactionEnable not found in backupmeta"
+test_log="new_collotion_enable_test.log"
+unset BR_LOG_TO_TERM
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --log-file $test_log || restore_fail=1
+if [ $restore_fail -ne 1 ]; then
+    echo "TEST: [$TEST_NAME] test restore failed!"
+    exit 1
+fi
+
+if ! grep -i "$error_str" $test_log; then
+    echo "${error_str} not found in log"
+    echo "TEST: [$TEST_NAME] test restore failed!"
+    exit 1
+fi
+
+rm -rf "$test_log"
+
+# backup with NewCollationEable = false
+echo "Restart cluster with new_collation_enable=false"
+start_services --tidb-cfg $cur/config/new_collation_enable_false.toml
+
+echo "backup start ... witch NewCollactionEnable=false in TiDB"
+run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$cur/${DB}_2"
+
+echo "Restart cluster with new_collation_enable=true"
+start_services --tidb-cfg $cur/config/new_collation_enable_true.toml
+
+echo "restore start ... with NewCollactionEnable=True in TiDB"
+restore_fail=0
+test_log2="new_collotion_enable_test2.log"
+error_str="newCollationEnable not match"
+unset BR_LOG_TO_TERM
+run_br restore db --db $DB -s "local://$cur/${DB}_2" --pd $PD_ADDR --log-file $test_log2 || restore_fail=1
+if [ $restore_fail -ne 1 ]; then
+    echo "TEST: [$TEST_NAME] test restore failed!"
+    exit 1
+fi
+
+if ! grep -i "$error_str" $test_log2; then
+    echo "${error_str} not found in log"
+    echo "TEST: [$TEST_NAME] test restore failed!"
+    exit 1
+fi
+
+rm -rf "$test_log2"
+rm -rf "$cur/${DB}_2"

br/tests/br_s3/run.sh

@@ -100,6 +100,12 @@ for p in $(seq 2); do
       exit 1
   fi
 
+  target_log="get newCollationEnable for check during restore"
+  if ! grep -i "$target_log" $BACKUP_LOG; then
+      echo "${target_log} not found in log"
+      exit 1
+  fi
+
   for i in $(seq $DB_COUNT); do
       run_sql "DROP DATABASE $DB${i};"
   done

executor/brie.go

@@ -480,6 +480,11 @@ func (gs *tidbGlueSession) CreateTable(ctx context.Context, dbName model.CIStr,
 func (gs *tidbGlueSession) Close() {
 }
 
+// GetGlobalVariables implements glue.Session.
+func (gs *tidbGlueSession) GetGlobalVariable(name string) (string, error) {
+	return gs.se.GetSessionVars().GlobalVarsAccessor.GetTiDBTableValue(name)
+}
+
 // Open implements glue.Glue
 func (gs *tidbGlueSession) Open(string, pd.SecurityOption) (kv.Storage, error) {
 	return gs.se.GetStore(), nil