ddl: add a binlog when dropping column (#13536) (#13613) (#13974)

pingcap · Dec 20, 2019 · 98a0ce6 · 98a0ce6
1 parent 547a855
commit 98a0ce6
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 11 deletions.
diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go
@@ -24,6 +24,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/terror"
+	pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta"
 	"github.com/pingcap/tidb/metrics"
@@ -396,9 +397,7 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error {
 			if err = w.handleUpdateJobError(t, job, err); err != nil {
 				return errors.Trace(err)
 			}
-			if job.IsDone() || job.IsRollbackDone() {
-				binloginfo.SetDDLBinlog(d.binlogCli, txn, job.ID, job.Query)
-			}
+			writeBinlog(d.binlogCli, txn, job)
 			return nil
 		})
 
@@ -431,6 +430,16 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error {
 	}
 }
 
+func writeBinlog(binlogCli *pumpcli.PumpsClient, txn kv.Transaction, job *model.Job) {
+	if job.IsDone() || job.IsRollbackDone() ||
+		// When this column is in the "delete only" and "delete reorg" states, the binlog of "drop column" has not been written yet,
+		// but the column has been removed from the binlog of the write operation.
+		// So we add this binlog to enable downstream components to handle DML correctly in this schema state.
+		(job.Type == model.ActionDropColumn && job.SchemaState == model.StateDeleteOnly) {
+		binloginfo.SetDDLBinlog(binlogCli, txn, job.ID, int32(job.SchemaState), job.Query)
+	}
+}
+
 // waitDependencyJobFinished waits for the dependency-job to be finished.
 // If the dependency job isn't finished yet, we'd better wait a moment.
 func (w *worker) waitDependencyJobFinished(job *model.Job, cnt *int) {

diff --git a/go.mod b/go.mod
@@ -55,7 +55,7 @@ require (
 	github.com/pingcap/parser v0.0.0-20191118062231-d32e257db9f2
 	github.com/pingcap/pd v2.1.12+incompatible
 	github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible
-	github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93
+	github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6
 	github.com/prometheus/client_golang v0.8.0
 	github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5
 	github.com/prometheus/common v0.0.0-20180426121432-d811d2e9bf89 // indirect

diff --git a/go.sum b/go.sum
@@ -115,8 +115,8 @@ github.com/pingcap/pd v2.1.12+incompatible h1:6N3LBxx2aSZqT+IWEG730EDNDttP7dXO8J
 github.com/pingcap/pd v2.1.12+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
 github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible h1:e9Gi/LP9181HT3gBfSOeSBA+5JfemuE4aEAhqNgoE4k=
 github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
-github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 h1:gI5bOzLMxjUq6ui+md/JnT4pYpkzrABJ/PeYORWiYYs=
-github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
+github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6 h1:HPgqtaqIFIZXTvQNiZoJ9Y79AXz3pmDpYFL28KraTKE=
+github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

diff --git a/sessionctx/binloginfo/binloginfo.go b/sessionctx/binloginfo/binloginfo.go
@@ -192,17 +192,18 @@ func (info *BinlogInfo) WriteBinlog(clusterID uint64) error {
 }
 
 // SetDDLBinlog sets DDL binlog in the kv.Transaction.
-func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery string) {
+func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlSchemaState int32, ddlQuery string) {
 	if client == nil {
 		return
 	}
 
 	ddlQuery = AddSpecialComment(ddlQuery)
 	info := &BinlogInfo{
 		Data: &binlog.Binlog{
-			Tp:       binlog.BinlogType_Prewrite,
-			DdlJobId: jobID,
-			DdlQuery: []byte(ddlQuery),
+			Tp:             binlog.BinlogType_Prewrite,
+			DdlJobId:       jobID,
+			DdlSchemaState: ddlSchemaState,
+			DdlQuery:       []byte(ddlQuery),
 		},
 		Client: client,
 	}

diff --git a/statistics/cmsketch_test.go b/statistics/cmsketch_test.go
@@ -122,7 +122,7 @@ func (s *testStatisticsSuite) TestCMSketchCoding(c *C) {
 	}
 	bytes, err := encodeCMSketch(lSketch)
 	c.Assert(err, IsNil)
-	c.Assert(len(bytes), Equals, 61455)
+	c.Assert(len(bytes), Equals, 61457)
 	rSketch, err := decodeCMSketch(bytes)
 	c.Assert(err, IsNil)
 	c.Assert(lSketch.Equal(rSketch), IsTrue)