Merge branch 'master' into ttl_metrics

br/pkg/storage/memstore_test.go

@@ -17,7 +17,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"sync"
 	"testing"
 	"time"
@@ -70,7 +69,7 @@ func TestMemStoreBasic(t *testing.T) {
 	require.Nil(t, err)
 	r2, err := store.Open(ctx, "/hello.txt")
 	require.Nil(t, err)
-	fileContent, err = ioutil.ReadAll(r)
+	fileContent, err = io.ReadAll(r)
 	require.Nil(t, err)
 	require.True(t, bytes.Equal([]byte("hello world 3"), fileContent))
 	require.Nil(t, r.Close())
@@ -83,7 +82,7 @@ func TestMemStoreBasic(t *testing.T) {
 
 	_, err = r2.Seek(5, io.SeekStart)
 	require.Nil(t, err)
-	fileContent, err = ioutil.ReadAll(r2)
+	fileContent, err = io.ReadAll(r2)
 	require.Nil(t, err)
 	require.True(t, bytes.Equal([]byte(" world 3"), fileContent))
 

cmd/explaintest/r/index_merge.result

@@ -455,9 +455,9 @@ c1	c2	c3
 ///// MEMORY Table
 explain select count(c1) from (select /*+ use_index_merge(t_alias), stream_agg() */ count(1) c1 from information_schema.statements_summary where sum_latency >= 0 or max_latency >= 0 order by 1) dt;
 id	estRows	task	access object	operator info
-StreamAgg_10	1.00	root		funcs:count(Column#93)->Column#94
-└─Sort_11	1.00	root		Column#93
-  └─StreamAgg_14	1.00	root		funcs:count(1)->Column#93
+StreamAgg_10	1.00	root		funcs:count(Column#96)->Column#97
+└─Sort_11	1.00	root		Column#96
+  └─StreamAgg_14	1.00	root		funcs:count(1)->Column#96
     └─MemTableScan_18	10000.00	root	table:STATEMENTS_SUMMARY	
 show warnings;
 Level	Code	Message

ddl/backfilling.go

@@ -626,8 +626,7 @@ func loadDDLReorgVars(ctx context.Context, sessPool *sessionPool) error {
 	return ddlutil.LoadDDLReorgVars(ctx, sCtx)
 }
 
-func makeupDecodeColMap(sessCtx sessionctx.Context, t table.Table) (map[int64]decoder.Column, error) {
-	dbName := model.NewCIStr(sessCtx.GetSessionVars().CurrentDB)
+func makeupDecodeColMap(sessCtx sessionctx.Context, dbName model.CIStr, t table.Table) (map[int64]decoder.Column, error) {
 	writableColInfos := make([]*model.ColumnInfo, 0, len(t.WritableCols()))
 	for _, col := range t.WritableCols() {
 		writableColInfos = append(writableColInfos, col.ColumnInfo)
@@ -860,7 +859,7 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sessionPool, t table.Physic
 
 	startKey, endKey := reorgInfo.StartKey, reorgInfo.EndKey
 	sessCtx := newContext(reorgInfo.d.store)
-	decodeColMap, err := makeupDecodeColMap(sessCtx, t)
+	decodeColMap, err := makeupDecodeColMap(sessCtx, reorgInfo.dbInfo.Name, t)
 	if err != nil {
 		return errors.Trace(err)
 	}

ddl/column.go

@@ -807,7 +807,11 @@ func doReorgWorkForModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.J
 	oldCol, changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) (done bool, ver int64, err error) {
 	job.ReorgMeta.ReorgTp = model.ReorgTypeTxn
 	rh := newReorgHandler(t, w.sess, w.concurrentDDL)
-	reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, tbl, BuildElements(changingCol, changingIdxs), false)
+	dbInfo, err := t.GetDatabase(job.SchemaID)
+	if err != nil {
+		return false, ver, errors.Trace(err)
+	}
+	reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, dbInfo, tbl, BuildElements(changingCol, changingIdxs), false)
 	if err != nil || reorgInfo.first {
 		// If we run reorg firstly, we should update the job snapshot version
 		// and then run the reorg next time.

ddl/db_change_test.go

@@ -1712,6 +1712,14 @@ func TestCreateExpressionIndex(t *testing.T) {
 	require.NoError(t, checkErr)
 	tk.MustExec("admin check table t")
 	tk.MustQuery("select * from t order by a, b").Check(testkit.Rows("0 9", "0 11", "0 11", "1 7", "2 7", "5 7", "8 8", "10 10", "10 10"))
+
+	// https://github.com/pingcap/tidb/issues/39784
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(name varchar(20))")
+	tk.MustExec("insert into t values ('Abc'), ('Bcd'), ('abc')")
+	tk.MustExec("create index idx on test.t((lower(test.t.name)))")
+	tk.MustExec("admin check table t")
 }
 
 func TestCreateUniqueExpressionIndex(t *testing.T) {

ddl/index.go

@@ -870,7 +870,11 @@ func runReorgJobAndHandleErr(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job,
 	tbl table.Table, indexInfo *model.IndexInfo, mergingTmpIdx bool) (done bool, ver int64, err error) {
 	elements := []*meta.Element{{ID: indexInfo.ID, TypeKey: meta.IndexElementKey}}
 	rh := newReorgHandler(t, w.sess, w.concurrentDDL)
-	reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, tbl, elements, mergingTmpIdx)
+	dbInfo, err := t.GetDatabase(job.SchemaID)
+	if err != nil {
+		return false, ver, errors.Trace(err)
+	}
+	reorgInfo, err := getReorgInfo(d.jobContext(job), d, rh, job, dbInfo, tbl, elements, mergingTmpIdx)
 	if err != nil || reorgInfo.first {
 		// If we run reorg firstly, we should update the job snapshot version
 		// and then run the reorg next time.

ddl/partition.go

@@ -1743,6 +1743,10 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
 		if err != nil {
 			return ver, errors.Trace(err)
 		}
+		dbInfo, err := t.GetDatabase(job.SchemaID)
+		if err != nil {
+			return ver, errors.Trace(err)
+		}
 		// If table has global indexes, we need reorg to clean up them.
 		if pt, ok := tbl.(table.PartitionedTable); ok && hasGlobalIndex(tblInfo) {
 			// Build elements for compatible with modify column type. elements will not be used when reorganizing.
@@ -1753,7 +1757,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
 				}
 			}
 			rh := newReorgHandler(t, w.sess, w.concurrentDDL)
-			reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job), d, rh, job, tbl, physicalTableIDs, elements)
+			reorgInfo, err := getReorgInfoFromPartitions(d.jobContext(job), d, rh, job, dbInfo, tbl, physicalTableIDs, elements)
 
 			if err != nil || reorgInfo.first {
 				// If we run reorg firstly, we should update the job snapshot version

ddl/reorg.go

@@ -386,6 +386,7 @@ type reorgInfo struct {
 	// PhysicalTableID is used to trace the current partition we are handling.
 	// If the table is not partitioned, PhysicalTableID would be TableID.
 	PhysicalTableID int64
+	dbInfo          *model.DBInfo
 	elements        []*meta.Element
 	currElement     *meta.Element
 }
@@ -585,7 +586,7 @@ func getValidCurrentVersion(store kv.Storage) (ver kv.Version, err error) {
 	return ver, nil
 }
 
-func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
+func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo,
 	tbl table.Table, elements []*meta.Element, mergingTmpIdx bool) (*reorgInfo, error) {
 	var (
 		element *meta.Element
@@ -685,11 +686,12 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job,
 	info.currElement = element
 	info.elements = elements
 	info.mergingTmpIdx = mergingTmpIdx
+	info.dbInfo = dbInfo
 
 	return &info, nil
 }
 
-func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, tbl table.Table, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) {
+func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, dbInfo *model.DBInfo, tbl table.Table, partitionIDs []int64, elements []*meta.Element) (*reorgInfo, error) {
 	var (
 		element *meta.Element
 		start   kv.Key
@@ -745,6 +747,7 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo
 	info.PhysicalTableID = pid
 	info.currElement = element
 	info.elements = elements
+	info.dbInfo = dbInfo
 
 	return &info, nil
 }

executor/set_test.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"strconv"
 	"testing"
@@ -1802,7 +1801,7 @@ func TestSetClusterConfig(t *testing.T) {
 	httpCnt = 0
 	tk.Session().SetValue(executor.TestSetConfigHTTPHandlerKey, func(req *http.Request) (*http.Response, error) {
 		httpCnt++
-		body, err := ioutil.ReadAll(req.Body)
+		body, err := io.ReadAll(req.Body)
 		require.NoError(t, err)
 		// The `raftstore.` prefix is stripped.
 		require.JSONEq(t, `{"server.snap-max-write-bytes-per-sec":"500MB"}`, string(body))

infoschema/tables.go

@@ -1323,6 +1323,9 @@ var tableStatementsSummaryCols = []columnInfo{
 	{name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"},
 	{name: stmtsummary.PlanStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Sampled execution plan"},
 	{name: stmtsummary.BinaryPlan, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Sampled binary plan"},
+	{name: stmtsummary.Charset, tp: mysql.TypeVarchar, size: 64, comment: "Sampled charset"},
+	{name: stmtsummary.Collation, tp: mysql.TypeVarchar, size: 64, comment: "Sampled collation"},
+	{name: stmtsummary.PlanHint, tp: mysql.TypeVarchar, size: 64, comment: "Sampled plan hint"},
 }
 
 var tableStorageStatsCols = []columnInfo{

infoschema/tables_test.go

@@ -1661,3 +1661,33 @@ func TestMemoryUsageAndOpsHistory(t *testing.T) {
 	require.Equal(t, row[10], "e3237ec256015a3566757e0c2742507cd30ae04e4cac2fbc14d269eafe7b067b")                                // SQL_DIGEST
 	require.Equal(t, row[11], "explain analyze select * from t t1 join t t2 join t t3 on t1.a=t2.a and t1.a=t3.a order by t1.a") // SQL_TEXT
 }
+
+func TestAddFieldsForBinding(t *testing.T) {
+	s := new(clusterTablesSuite)
+	s.store, s.dom = testkit.CreateMockStoreAndDomain(t)
+	s.rpcserver, s.listenAddr = s.setUpRPCService(t, "127.0.0.1:0", nil)
+	s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer()
+	s.startTime = time.Now()
+	defer s.httpServer.Close()
+	defer s.rpcserver.Stop()
+	tk := s.newTestKitWithRoot(t)
+
+	require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil))
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, key(a))")
+	tk.MustExec("select /*+ ignore_index(t, a)*/ * from t where a = 1")
+	planDigest := "4e3159169cc63c14b139a4e7d72eae1759875c9a9581f94bb2079aae961189cb"
+	rows := tk.MustQuery(fmt.Sprintf("select stmt_type, prepared, sample_user, schema_name, query_sample_text, charset, collation, plan_hint, digest_text "+
+		"from information_schema.cluster_statements_summary where plan_digest = '%s'", planDigest)).Rows()
+
+	require.Equal(t, rows[0][0], "Select")
+	require.Equal(t, rows[0][1], "0")
+	require.Equal(t, rows[0][2], "root")
+	require.Equal(t, rows[0][3], "test")
+	require.Equal(t, rows[0][4], "select /*+ ignore_index(t, a)*/ * from t where a = 1")
+	require.Equal(t, rows[0][5], "utf8mb4")
+	require.Equal(t, rows[0][6], "utf8mb4_bin")
+	require.Equal(t, rows[0][7], "use_index(@`sel_1` `test`.`t` ), ignore_index(`t` `a`)")
+	require.Equal(t, rows[0][8], "select * from `t` where `a` = ?")
+}

parser/consistent_test.go

@@ -14,7 +14,7 @@
 package parser
 
 import (
-	"io/ioutil"
+	gio "io"
 	"os"
 	"sort"
 	"strings"
@@ -27,7 +27,7 @@ func TestKeywordConsistent(t *testing.T) {
 	parserFilename := "parser.y"
 	parserFile, err := os.Open(parserFilename)
 	requires.NoError(t, err)
-	data, err := ioutil.ReadAll(parserFile)
+	data, err := gio.ReadAll(parserFile)
 	requires.NoError(t, err)
 	content := string(data)
 

parser/format/format_test.go

@@ -15,7 +15,7 @@ package format
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"strings"
 	"testing"
 
@@ -26,7 +26,7 @@ import (
 func checkFormat(t *testing.T, f Formatter, buf *bytes.Buffer, str, expect string) {
 	_, err := f.Format(str, 3)
 	require.NoError(t, err)
-	b, err := ioutil.ReadAll(buf)
+	b, err := io.ReadAll(buf)
 	require.NoError(t, err)
 	require.Equal(t, expect, string(b))
 }

parser/reserved_words_test.go

@@ -28,7 +28,7 @@ import (
 	// needed to connect to MySQL
 
 	dbsql "database/sql"
-	"io/ioutil"
+	gio "io"
 	"os"
 	"testing"
 
@@ -41,7 +41,7 @@ func TestCompareReservedWordsWithMySQL(t *testing.T) {
 	parserFilename := "parser.y"
 	parserFile, err := os.Open(parserFilename)
 	requires.NoError(t, err)
-	data, err := ioutil.ReadAll(parserFile)
+	data, err := gio.ReadAll(parserFile)
 	requires.NoError(t, err)
 	content := string(data)
 

planner/core/binary_plan_test.go

@@ -17,7 +17,7 @@ package core_test
 import (
 	"encoding/base64"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"regexp"
 	"strings"
@@ -521,7 +521,7 @@ func TestUnnecessaryBinaryPlanInSlowLog(t *testing.T) {
 	tk.MustExec("drop table if exists th")
 	tk.MustExec("set global tidb_slow_log_threshold = 1;")
 	tk.MustExec("create table th (i int, a int,b int, c int, index (a)) partition by hash (a) partitions 100;")
-	slowLogBytes, err := ioutil.ReadAll(f)
+	slowLogBytes, err := io.ReadAll(f)
 	require.NoError(t, err)
 	require.NotContains(t, string(slowLogBytes), `tidb_decode_binary_plan('')`)
 }

server/plan_replayer.go

@@ -16,7 +16,7 @@ package server
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -83,7 +83,7 @@ func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req
 			writeError(w, err)
 			return
 		}
-		content, err := ioutil.ReadAll(file)
+		content, err := io.ReadAll(file)
 		if err != nil {
 			writeError(w, err)
 			return
@@ -137,7 +137,7 @@ func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req
 				zap.String("remote-addr", remoteAddr), zap.Int("status-code", resp.StatusCode))
 			continue
 		}
-		content, err := ioutil.ReadAll(resp.Body)
+		content, err := io.ReadAll(resp.Body)
 		if err != nil {
 			writeError(w, err)
 			return

server/plan_replayer_test.go

@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"database/sql"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -69,7 +68,7 @@ func TestDumpPlanReplayerAPI(t *testing.T) {
 		require.NoError(t, resp0.Body.Close())
 	}()
 
-	body, err := ioutil.ReadAll(resp0.Body)
+	body, err := io.ReadAll(resp0.Body)
 	require.NoError(t, err)
 
 	path := "/tmp/plan_replayer.zip"

statistics/handle/dump.go

@@ -19,7 +19,7 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"time"
 
 	"github.com/pingcap/errors"
@@ -501,7 +501,7 @@ func BlocksToJSONTable(blocks [][]byte) (*JSONTable, error) {
 	if err := gzipReader.Close(); err != nil {
 		return nil, err
 	}
-	jsonStr, err := ioutil.ReadAll(gzipReader)
+	jsonStr, err := io.ReadAll(gzipReader)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}

util/cpuprofile/cpuprofile_test.go

@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"runtime/pprof"
@@ -237,7 +236,7 @@ func TestProfileHTTPHandler(t *testing.T) {
 	resp, err = http.Get("http://" + address + "/debug/pprof/profile?seconds=100000")
 	require.NoError(t, err)
 	require.Equal(t, 400, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.Equal(t, "profile duration exceeds server's WriteTimeout\n", string(body))
 	require.NoError(t, resp.Body.Close())

util/stmtsummary/reader.go

@@ -310,6 +310,9 @@ const (
 	PlanDigestStr                     = "PLAN_DIGEST"
 	PlanStr                           = "PLAN"
 	BinaryPlan                        = "BINARY_PLAN"
+	Charset                           = "CHARSET"
+	Collation                         = "COLLATION"
+	PlanHint                          = "PLAN_HINT"
 )
 
 type columnValueFactory func(reader *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest) interface{}
@@ -620,4 +623,13 @@ var columnValueFactoryMap = map[string]columnValueFactory{
 	BinaryPlan: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
 		return ssElement.sampleBinaryPlan
 	},
+	Charset: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return ssElement.charset
+	},
+	Collation: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return ssElement.collation
+	},
+	PlanHint: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return ssElement.planHint
+	},
 }