ttl: make the TTL compatible with dumpling, lightning and BR

br/pkg/lightning/restore/tidb.go

@@ -152,7 +152,7 @@ func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string
 	}
 
 	var res strings.Builder
-	ctx := format.NewRestoreCtx(format.DefaultRestoreFlags|format.RestoreTiDBSpecialComment, &res)
+	ctx := format.NewRestoreCtx(format.DefaultRestoreFlags|format.RestoreTiDBSpecialComment|format.RestoreWithTTLEnableOff, &res)
 
 	retStmts := make([]string, 0, len(stmts))
 	for _, stmt := range stmts {

br/pkg/restore/db.go

@@ -308,6 +308,10 @@ func (db *DB) CreateTables(ctx context.Context, tables []*metautil.Table,
 					return errors.Trace(err)
 				}
 			}
+
+			if ttlInfo := table.Info.TTLInfo; ttlInfo != nil {
+				ttlInfo.Enable = false
+			}
 		}
 		if err := batchSession.CreateTables(ctx, m, db.tableIDAllocFilter()); err != nil {
 			return err
@@ -336,6 +340,10 @@ func (db *DB) CreateTable(ctx context.Context, table *metautil.Table,
 		}
 	}
 
+	if ttlInfo := table.Info.TTLInfo; ttlInfo != nil {
+		ttlInfo.Enable = false
+	}
+
 	err := db.se.CreateTable(ctx, table.DB.Name, table.Info, db.tableIDAllocFilter())
 	if err != nil {
 		log.Error("create table failed",

br/pkg/stream/BUILD.bazel

@@ -56,8 +56,11 @@ go_test(
         "//br/pkg/storage",
         "//br/pkg/streamhelper",
         "//meta",
+        "//parser/ast",
         "//parser/model",
+        "//parser/mysql",
         "//tablecodec",
+        "//types",
         "//util/codec",
         "//util/table-filter",
         "@com_github_pingcap_kvproto//pkg/brpb",

br/pkg/stream/rewrite_meta_rawkv.go

@@ -336,6 +336,11 @@ func (sr *SchemasReplace) rewriteTableInfo(value []byte, dbID int64) ([]byte, bo
 		}
 	}
 
+	// Force to disable TTL_ENABLE when restore
+	if newTableInfo.TTLInfo != nil {
+		newTableInfo.TTLInfo.Enable = false
+	}
+
 	if sr.AfterTableRewritten != nil {
 		sr.AfterTableRewritten(false, newTableInfo)
 	}

br/pkg/stream/rewrite_meta_rawkv_test.go

@@ -7,7 +7,10 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/pingcap/tidb/parser/ast"
 	"github.com/pingcap/tidb/parser/model"
+	"github.com/pingcap/tidb/parser/mysql"
+	"github.com/pingcap/tidb/types"
 	filter "github.com/pingcap/tidb/util/table-filter"
 	"github.com/stretchr/testify/require"
 )
@@ -312,6 +315,52 @@ func TestRewriteValueForExchangePartition(t *testing.T) {
 	require.Equal(t, tableInfo.ID, pt1ID+100)
 }
 
+func TestRewriteValueForTTLTable(t *testing.T) {
+	var (
+		dbId      int64 = 40
+		tableID   int64 = 100
+		colID     int64 = 1000
+		colName         = "t"
+		tableName       = "t1"
+		tableInfo model.TableInfo
+	)
+
+	tbl := model.TableInfo{
+		ID:   tableID,
+		Name: model.NewCIStr(tableName),
+		Columns: []*model.ColumnInfo{
+			{
+				ID:        colID,
+				Name:      model.NewCIStr(colName),
+				FieldType: *types.NewFieldType(mysql.TypeTimestamp),
+			},
+		},
+		TTLInfo: &model.TTLInfo{
+			ColumnName:       model.NewCIStr(colName),
+			IntervalExprStr:  "1",
+			IntervalTimeUnit: int(ast.TimeUnitDay),
+			Enable:           true,
+		},
+	}
+	value, err := json.Marshal(&tbl)
+	require.Nil(t, err)
+
+	sr := MockEmptySchemasReplace(nil)
+	newValue, needRewrite, err := sr.rewriteTableInfo(value, dbId)
+	require.Nil(t, err)
+	require.True(t, needRewrite)
+
+	err = json.Unmarshal(newValue, &tableInfo)
+	require.Nil(t, err)
+	require.Equal(t, tableInfo.Name.String(), tableName)
+	require.Equal(t, tableInfo.ID, sr.DbMap[dbId].TableMap[tableID].NewTableID)
+	require.NotNil(t, tableInfo.TTLInfo)
+	require.Equal(t, colName, tableInfo.TTLInfo.ColumnName.O)
+	require.Equal(t, "1", tableInfo.TTLInfo.IntervalExprStr)
+	require.Equal(t, int(ast.TimeUnitDay), tableInfo.TTLInfo.IntervalTimeUnit)
+	require.False(t, tableInfo.TTLInfo.Enable)
+}
+
 // db:70->80 -
 //           | - t0:71->81 -
 //           |             | - p0:72->82

br/tests/br_ttl/run.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+PROGRESS_FILE="$TEST_DIR/progress_file"
+BACKUPMETAV1_LOG="$TEST_DIR/backup.log"
+BACKUPMETAV2_LOG="$TEST_DIR/backupv2.log"
+RESTORE_LOG="$TEST_DIR/restore.log"
+rm -rf $PROGRESS_FILE
+
+run_sql "create schema $DB;"
+run_sql "create table $DB.ttl_test_tbl(id int primary key, t datetime) TTL=\`t\` + interval 1 day TTL_ENABLE='ON'"
+
+# backup db
+echo "full backup meta v2 start..."
+unset BR_LOG_TO_TERM
+rm -f $BACKUPMETAV2_LOG
+run_br backup full --log-file $BACKUPMETAV2_LOG -s "local://$TEST_DIR/${DB}v2" --pd $PD_ADDR --use-backupmeta-v2
+
+echo "full backup meta v1 start..."
+rm -f $BACKUPMETAV1_LOG
+run_br backup full --log-file $BACKUPMETAV1_LOG -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+TTL_MARK='![ttl]'
+CREATE_SQL_CONTAINS="/*T${TTL_MARK} TTL=\`t\` + INTERVAL 1 DAY */ /*T${TTL_MARK} TTL_ENABLE='OFF' */"
+
+# restore v2
+run_sql "DROP DATABASE $DB;"
+echo "restore ttl table start v2..."
+run_br restore db --db $DB -s "local://$TEST_DIR/${DB}v2" --pd $PD_ADDR
+run_sql "show create table $DB.ttl_test_tbl;"
+check_contains "$CREATE_SQL_CONTAINS"
+
+# restore v1
+run_sql "DROP DATABASE $DB;"
+echo "restore ttl table start v1..."
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+run_sql "show create table $DB.ttl_test_tbl;"
+check_contains "$CREATE_SQL_CONTAINS"

br/tests/lightning_ttl/config.toml

@@ -0,0 +1,2 @@
+[tikv-importer]
+backend = 'local'

br/tests/lightning_ttl/data/ttldb-schema-create.sql

@@ -0,0 +1 @@
+CREATE DATABASE `ttldb`;

br/tests/lightning_ttl/data/ttldb.t1-schema.sql

@@ -0,0 +1,4 @@
+CREATE TABLE `t1` (
+    `id` int(11) PRIMARY KEY,
+    `t` datetime
+) TTL = `t` + INTERVAL 1 DAY TTL_ENABLE = 'ON';

br/tests/lightning_ttl/run.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+run_sql 'drop database if exists ttldb;'
+run_lightning
+
+TTL_MARK='![ttl]'
+CREATE_SQL_CONTAINS="/*T${TTL_MARK} TTL=\`t\` + INTERVAL 1 DAY */ /*T${TTL_MARK} TTL_ENABLE='OFF' */"
+
+run_sql 'show create table ttldb.t1'
+check_contains "$CREATE_SQL_CONTAINS"

parser/ast/ddl.go

@@ -1096,7 +1096,8 @@ func (n *CreateTableStmt) Restore(ctx *format.RestoreCtx) error {
 		ctx.WritePlain(")")
 	}
 
-	for i, option := range n.Options {
+	options := tableOptionsWithRestoreTTLFlag(ctx.Flags, n.Options)
+	for i, option := range options {
 		ctx.WritePlain(" ")
 		if err := option.Restore(ctx); err != nil {
 			return errors.Annotatef(err, "An error occurred while splicing CreateTableStmt TableOption: [%v]", i)
@@ -3573,11 +3574,21 @@ func (n *AlterTableStmt) Restore(ctx *format.RestoreCtx) error {
 	if err := n.Table.Restore(ctx); err != nil {
 		return errors.Annotate(err, "An error occurred while restore AlterTableStmt.Table")
 	}
-	var specs []*AlterTableSpec
+	specs := make([]*AlterTableSpec, 0, len(n.Specs))
 	for _, spec := range n.Specs {
-		if !(spec.IsAllPlacementRule() && ctx.Flags.HasSkipPlacementRuleForRestoreFlag()) {
-			specs = append(specs, spec)
+		if spec.IsAllPlacementRule() && ctx.Flags.HasSkipPlacementRuleForRestoreFlag() {
+			continue
+		}
+		if spec.Tp == AlterTableOption {
+			newOptions := tableOptionsWithRestoreTTLFlag(ctx.Flags, spec.Options)
+			if len(newOptions) == 0 {
+				continue
+			}
+			newSpec := *spec
+			newSpec.Options = newOptions
+			spec = &newSpec
 		}
+		specs = append(specs, spec)
 	}
 	for i, spec := range specs {
 		if i == 0 || spec.Tp == AlterTablePartition || spec.Tp == AlterTableRemovePartitioning || spec.Tp == AlterTableImportTablespace || spec.Tp == AlterTableDiscardTablespace {
@@ -4510,3 +4521,25 @@ func restorePlacementStmtInSpecialComment(ctx *format.RestoreCtx, n DDLNode) err
 		return n.Restore(ctx)
 	})
 }
+
+func tableOptionsWithRestoreTTLFlag(flags format.RestoreFlags, options []*TableOption) []*TableOption {
+	if !flags.HasRestoreWithTTLEnableOff() {
+		return options
+	}
+
+	newOptions := make([]*TableOption, 0, len(options))
+	for _, opt := range options {
+		if opt.Tp == TableOptionTTLEnable {
+			continue
+		}
+
+		newOptions = append(newOptions, opt)
+		if opt.Tp == TableOptionTTL {
+			newOptions = append(newOptions, &TableOption{
+				Tp:        TableOptionTTLEnable,
+				BoolValue: false,
+			})
+		}
+	}
+	return newOptions
+}

parser/ast/ddl_test.go

@@ -869,3 +869,37 @@ func TestTableOptionTTLRestore(t *testing.T) {
 		runNodeRestoreTestWithFlags(t, testCases, "%s", extractNodeFunc, ca.flags)
 	}
 }
+
+func TestTableOptionTTLRestoreWithTTLEnableOffFlag(t *testing.T) {
+	sourceSQL1 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR"
+	sourceSQL2 := "alter table t ttl_enable = 'ON'"
+	sourceSQL3 := "alter table t remove ttl"
+	sourceSQL4 := "create table t (created_at datetime) ttl = created_at + INTERVAL 1 YEAR ttl_enable = 'ON'"
+	sourceSQL5 := "alter table t ttl_enable = 'ON' placement policy p1"
+	cases := []struct {
+		sourceSQL string
+		flags     format.RestoreFlags
+		expectSQL string
+	}{
+		{sourceSQL1, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR TTL_ENABLE = 'OFF'"},
+		{sourceSQL1, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */ /*T![ttl] TTL_ENABLE = 'OFF' */"},
+		{sourceSQL2, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "ALTER TABLE `t`"},
+		{sourceSQL2, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t`"},
+		{sourceSQL3, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` REMOVE TTL"},
+		{sourceSQL3, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` /*T![ttl] REMOVE TTL */"},
+		{sourceSQL4, format.DefaultRestoreFlags | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) TTL = `created_at` + INTERVAL 1 YEAR TTL_ENABLE = 'OFF'"},
+		{sourceSQL4, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "CREATE TABLE `t` (`created_at` DATETIME) /*T![ttl] TTL = `created_at` + INTERVAL 1 YEAR */ /*T![ttl] TTL_ENABLE = 'OFF' */"},
+		{sourceSQL5, format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment | format.RestoreWithTTLEnableOff, "ALTER TABLE `t` /*T![placement] PLACEMENT POLICY = `p1` */"},
+	}
+
+	extractNodeFunc := func(node Node) Node {
+		return node
+	}
+
+	for _, ca := range cases {
+		testCases := []NodeRestoreTestCase{
+			{ca.sourceSQL, ca.expectSQL},
+		}
+		runNodeRestoreTestWithFlagsStmtChange(t, testCases, "%s", extractNodeFunc, ca.flags)
+	}
+}

parser/format/format.go

@@ -235,6 +235,7 @@ const (
 
 	RestoreTiDBSpecialComment
 	SkipPlacementRuleForRestore
+	RestoreWithTTLEnableOff
 )
 
 const (
@@ -321,6 +322,11 @@ func (rf RestoreFlags) HasSkipPlacementRuleForRestoreFlag() bool {
 	return rf.has(SkipPlacementRuleForRestore)
 }
 
+// HasRestoreWithTTLEnableOff returns a boolean indicating whether to force set TTL_ENABLE='OFF' when restoring a TTL table
+func (rf RestoreFlags) HasRestoreWithTTLEnableOff() bool {
+	return rf.has(RestoreWithTTLEnableOff)
+}
+
 // RestoreCtx is `Restore` context to hold flags and writer.
 type RestoreCtx struct {
 	Flags     RestoreFlags