*: reduce dumpling accessing database and information_schema usage to…

… improve its stability (#305) (#315)

Makefile

@@ -96,6 +96,7 @@ static: tools
 		--disable gosec \
 		--disable errorlint \
 		--disable sqlclosecheck \
+		--disable scopelint \
 		$$($(PACKAGE_DIRECTORIES))
 	# pingcap/errors APIs are mixed with multiple patterns 'pkg/errors',
 	# 'juju/errors' and 'pingcap/parser'. To avoid confusion and mistake,

go.mod

@@ -22,9 +22,6 @@ require (
 	github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect
-	github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed // indirect
-	github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047
 	github.com/soheilhy/cmux v0.1.4
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -444,7 +444,6 @@ github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIf
 github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 h1:ERrF0fTuIOnwfGbt71Ji3DKbOEaP189tjym50u8gpC8=
 github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
 github.com/pingcap/parser v0.0.0-20210513020953-ae2c4497c07b h1:eLuDQ6eJCEKCbGwhGrkjzagwev1GJGU2Y2kFkAsBzV0=
 github.com/pingcap/parser v0.0.0-20210513020953-ae2c4497c07b/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
@@ -455,7 +454,6 @@ github.com/pingcap/tidb v1.1.0-beta.0.20210517044538-8ad868f801fc/go.mod h1:MTGi
 github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo=
 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible h1:ceznmu/lLseGHP/jKyOa/3u/5H3wtLLLqkH2V3ssSjg=
 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
-github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1 h1:Kcp3jIcQrqG+pT1JQ0oWyRncVKQtDgnMFzRt3zJBaBo=
 github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1/go.mod h1:nsEhnMokcn7MRqd2J60yxpn/ac3ZH8A6GOJ9NslabUo=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -504,30 +502,19 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM=
-github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil v3.21.2+incompatible h1:U+YvJfjCh6MslYlIAXvPtzhW3YZEtc9uncueUNpD/0A=
 github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk=
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
 github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0=
 github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
-github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM=
-github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
-github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
-github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed h1:KMgQoLJGCq1IoZpLZE3AIffh9veYWoVlsvA4ib55TMM=
-github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
-github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047 h1:boyJ8EgQN/aC3grvx8QUoJrptt7RvneezSJSCbW25a4=
-github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=

tests/run.sh

@@ -4,8 +4,6 @@
 
 # To avoid permission denied error, please run `chmod +x tests/_utils/*`.
 
-set -e
-
 DUMPLING_TEST_DIR=${DUMPLING_TEST_DIR:-"/tmp/dumpling_test_result"}
 DUMPLING_TEST_USER=${DUMPLING_TEST_USER:-"root"}
 

tests/views/run.sh

@@ -17,8 +17,18 @@ run_dumpling --no-views
 file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
 file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
 
+rm -rf $DUMPLING_OUTPUT_DIR
 run_dumpling --no-views=false
 #diff "$DUMPLING_BASE_NAME/data/views-schema-create.sql" "$DUMPLING_OUTPUT_DIR/views-schema-create.sql"
 diff "$DUMPLING_BASE_NAME/data/views.v-schema.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
 diff "$DUMPLING_BASE_NAME/data/views.v-schema-view.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
-file_not_exist "$DUMPLING_BASE_NAME/data/views.v.000000000.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v.000000000.sql"
+
+# test --no-schemas
+rm -rf $DUMPLING_OUTPUT_DIR
+run_dumpling --no-schemas
+file_not_exist "$DUMPLING_OUTPUT_DIR/views-schema-create.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.t-schema.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v.000000000.sql"

v4/export/block_allow_list.go

@@ -9,13 +9,17 @@ import (
 )
 
 func filterTables(tctx *tcontext.Context, conf *Config) {
+	filterTablesFunc(tctx, conf, conf.TableFilter.MatchTable)
+}
+
+func filterTablesFunc(tctx *tcontext.Context, conf *Config, matchTable func(string, string) bool) {
 	tctx.L().Debug("start to filter tables")
 	dbTables := DatabaseTables{}
 	ignoredDBTable := DatabaseTables{}
 
 	for dbName, tables := range conf.Tables {
 		for _, table := range tables {
-			if conf.TableFilter.MatchTable(dbName, table.Name) {
+			if matchTable(dbName, table.Name) {
 				dbTables.AppendTable(dbName, table)
 			} else {
 				ignoredDBTable.AppendTable(dbName, table)

v4/export/block_allow_list_test.go

@@ -20,11 +20,11 @@ func (s *testBWListSuite) TestFilterTables(c *C) {
 	dbTables := DatabaseTables{}
 	expectedDBTables := DatabaseTables{}
 
-	dbTables.AppendTables(filter.InformationSchemaName, []string{"xxx"}...)
-	dbTables.AppendTables(strings.ToUpper(filter.PerformanceSchemaName), []string{"xxx"}...)
-	dbTables.AppendTables("xxx", []string{"yyy"}...)
-	expectedDBTables.AppendTables("xxx", []string{"yyy"}...)
-	dbTables.AppendTables("yyy", []string{"xxx"}...)
+	dbTables.AppendTables(filter.InformationSchemaName, []string{"xxx"}, []uint64{0})
+	dbTables.AppendTables(strings.ToUpper(filter.PerformanceSchemaName), []string{"xxx"}, []uint64{0})
+	dbTables.AppendTables("xxx", []string{"yyy"}, []uint64{0})
+	expectedDBTables.AppendTables("xxx", []string{"yyy"}, []uint64{0})
+	dbTables.AppendTables("yyy", []string{"xxx"}, []uint64{0})
 
 	tableFilter, err := tf.Parse([]string{"*.*"})
 	c.Assert(err, IsNil)

v4/export/consistency.go

@@ -34,8 +34,8 @@ func NewConsistencyController(ctx context.Context, conf *Config, session *sql.DB
 		}, nil
 	case consistencyTypeLock:
 		return &ConsistencyLockDumpingTables{
-			conn:      conn,
-			allTables: conf.Tables,
+			conn: conn,
+			conf: conf,
 		}, nil
 	case consistencyTypeSnapshot:
 		if conf.ServerInfo.ServerType != ServerTypeTiDB {
@@ -110,16 +110,28 @@ func (c *ConsistencyFlushTableWithReadLock) PingContext(ctx context.Context) err
 
 // ConsistencyLockDumpingTables execute lock tables read on all tables before dump
 type ConsistencyLockDumpingTables struct {
-	conn      *sql.Conn
-	allTables DatabaseTables
+	conn *sql.Conn
+	conf *Config
 }
 
 // Setup implements ConsistencyController.Setup
 func (c *ConsistencyLockDumpingTables) Setup(tctx *tcontext.Context) error {
 	blockList := make(map[string]map[string]interface{})
 	return utils.WithRetry(tctx, func() error {
-		lockTablesSQL := buildLockTablesSQL(c.allTables, blockList)
+		lockTablesSQL := buildLockTablesSQL(c.conf.Tables, blockList)
 		_, err := c.conn.ExecContext(tctx, lockTablesSQL)
+		if err == nil {
+			if len(blockList) > 0 {
+				filterTablesFunc(tctx, c.conf, func(db string, tbl string) bool {
+					if blockTable, ok := blockList[db]; ok {
+						if _, ok := blockTable[tbl]; ok {
+							return false
+						}
+					}
+					return true
+				})
+			}
+		}
 		return errors.Trace(err)
 	}, newLockTablesBackoffer(tctx, blockList))
 }

v4/export/consistency_test.go

@@ -65,17 +65,15 @@ func (s *testConsistencySuite) TestConsistencyController(c *C) {
 
 	conf.Consistency = consistencyTypeLock
 	conf.Tables = NewDatabaseTables().
-		AppendTables("db1", "t1", "t2", "t3").
+		AppendTables("db1", []string{"t1", "t2", "t3"}, []uint64{1, 2, 3}).
 		AppendViews("db2", "t4")
 	mock.ExpectExec("LOCK TABLES `db1`.`t1` READ,`db1`.`t2` READ,`db1`.`t3` READ").WillReturnResult(resultOk)
 	mock.ExpectExec("UNLOCK TABLES").WillReturnResult(resultOk)
 	ctrl, _ = NewConsistencyController(ctx, conf, db)
 	_, ok = ctrl.(*ConsistencyLockDumpingTables)
 	c.Assert(ok, IsTrue)
 	s.assertLifetimeErrNil(tctx, ctrl, c)
-	if err = mock.ExpectationsWereMet(); err != nil {
-		c.Fatal(err.Error())
-	}
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
 
 func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
@@ -90,7 +88,7 @@ func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
 
 	conf.Consistency = consistencyTypeLock
 	conf.Tables = NewDatabaseTables().
-		AppendTables("db1", "t1", "t2", "t3").
+		AppendTables("db1", []string{"t1", "t2", "t3"}, []uint64{1, 2, 3}).
 		AppendViews("db2", "t4")
 	mock.ExpectExec("LOCK TABLES `db1`.`t1` READ,`db1`.`t2` READ,`db1`.`t3` READ").
 		WillReturnError(&mysql.MySQLError{Number: ErrNoSuchTable, Message: "Table 'db1.t3' doesn't exist"})
@@ -100,9 +98,12 @@ func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
 	_, ok := ctrl.(*ConsistencyLockDumpingTables)
 	c.Assert(ok, IsTrue)
 	s.assertLifetimeErrNil(tctx, ctrl, c)
-	if err = mock.ExpectationsWereMet(); err != nil {
-		c.Fatal(err.Error())
-	}
+	// should remove table db1.t3 in tables to dump
+	expectedDumpTables := NewDatabaseTables().
+		AppendTables("db1", []string{"t1", "t2"}, []uint64{1, 2}).
+		AppendViews("db2", "t4")
+	c.Assert(conf.Tables, DeepEquals, expectedDumpTables)
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
 
 func (s *testConsistencySuite) TestResolveAutoConsistency(c *C) {
@@ -156,7 +157,7 @@ func (s *testConsistencySuite) TestConsistencyControllerError(c *C) {
 
 	// lock table fail
 	conf.Consistency = consistencyTypeLock
-	conf.Tables = NewDatabaseTables().AppendTables("db", "t")
+	conf.Tables = NewDatabaseTables().AppendTables("db", []string{"t"}, []uint64{1})
 	mock.ExpectExec("LOCK TABLE").WillReturnError(errors.New(""))
 	ctrl, _ = NewConsistencyController(ctx, conf, db)
 	err = ctrl.Setup(tctx)