Skip to content

Commit 2cddfb3

Browse files
YisaerYisaer
authored
domain: support more params for extract plan task (#41975)
ref #41130
1 parent 60eceff commit 2cddfb3

File tree

7 files changed

+125
-21
lines changed

7 files changed

+125
-21
lines changed

‎domain/domain.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -977,7 +977,7 @@ func NewDomain(store kv.Storage, ddlLease time.Duration, statsLease time.Duratio
977977
infoCache: infoschema.NewCache(16),
978978
slowQuery: newTopNSlowQueries(30, time.Hour*24*7, 500),
979979
indexUsageSyncLease: idxUsageSyncLease,
980-
dumpFileGcChecker: &dumpFileGcChecker{gcLease: dumpFileGcLease, paths: []string{replayer.GetPlanReplayerDirName(), GetOptimizerTraceDirName()}},
980+
dumpFileGcChecker: &dumpFileGcChecker{gcLease: dumpFileGcLease, paths: []string{replayer.GetPlanReplayerDirName(), GetOptimizerTraceDirName(), GetExtractTaskDirName()}},
981981
expiredTimeStamp4PC: types.NewTime(types.ZeroCoreTime, mysql.TypeTimestamp, types.DefaultFsp),
982982
mdlCheckTableInfo: &mdlCheckTableInfo{
983983
mu: sync.Mutex{},

‎domain/extract.go

+32-10
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"math/rand"
2424
"os"
2525
"path/filepath"
26+
"strconv"
2627
"strings"
2728
"sync"
2829
"time"
@@ -48,6 +49,8 @@ const (
4849
const (
4950
// ExtractTaskType indicates type of extract task
5051
ExtractTaskType = "taskType"
52+
// ExtractPlanTaskSkipStats indicates skip stats for extract plan task
53+
ExtractPlanTaskSkipStats = "SkipStats"
5154
)
5255

5356
// ExtractType indicates type
@@ -102,6 +105,10 @@ type ExtractTask struct {
102105
ExtractType ExtractType
103106
IsBackgroundJob bool
104107

108+
// Param for Extract Plan
109+
SkipStats bool
110+
UseHistoryView bool
111+
105112
// variables for plan task type
106113
Begin time.Time
107114
End time.Time
@@ -132,7 +139,7 @@ func (w *extractWorker) extractTask(ctx context.Context, task *ExtractTask) (str
132139
}
133140

134141
func (w *extractWorker) extractPlanTask(ctx context.Context, task *ExtractTask) (string, error) {
135-
if !config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent {
142+
if task.UseHistoryView && !config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent {
136143
return "", errors.New("tidb_stmt_summary_enable_persistent should be enabled for extract task")
137144
}
138145
records, err := w.collectRecords(ctx, task)
@@ -145,16 +152,20 @@ func (w *extractWorker) extractPlanTask(ctx context.Context, task *ExtractTask)
145152
logutil.BgLogger().Error("package stmt summary records failed for extract plan task", zap.Error(err))
146153
return "", err
147154
}
148-
return w.dumpExtractPlanPackage(p)
155+
return w.dumpExtractPlanPackage(task, p)
149156
}
150157

151158
func (w *extractWorker) collectRecords(ctx context.Context, task *ExtractTask) (map[stmtSummaryHistoryKey]*stmtSummaryHistoryRecord, error) {
152159
w.Lock()
153160
defer w.Unlock()
154161
exec := w.sctx.(sqlexec.RestrictedSQLExecutor)
155162
ctx1 := kv.WithInternalSourceType(ctx, kv.InternalTxnStats)
156-
rows, _, err := exec.ExecRestrictedSQL(ctx1, nil, fmt.Sprintf("SELECT STMT_TYPE, DIGEST, PLAN_DIGEST,QUERY_SAMPLE_TEXT, BINARY_PLAN, TABLE_NAMES FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY WHERE SUMMARY_END_TIME > '%s' OR SUMMARY_BEGIN_TIME < '%s'",
157-
task.Begin.Format(types.TimeFormat), task.End.Format(types.TimeFormat)))
163+
sourceTable := "STATEMENTS_SUMMARY_HISTORY"
164+
if !task.UseHistoryView {
165+
sourceTable = "STATEMENTS_SUMMARY"
166+
}
167+
rows, _, err := exec.ExecRestrictedSQL(ctx1, nil, fmt.Sprintf("SELECT STMT_TYPE, DIGEST, PLAN_DIGEST,QUERY_SAMPLE_TEXT, BINARY_PLAN, TABLE_NAMES, SAMPLE_USER FROM INFORMATION_SCHEMA.%s WHERE SUMMARY_END_TIME > '%s' OR SUMMARY_BEGIN_TIME < '%s'",
168+
sourceTable, task.Begin.Format(types.TimeFormat), task.End.Format(types.TimeFormat)))
158169
if err != nil {
159170
return nil, err
160171
}
@@ -171,6 +182,7 @@ func (w *extractWorker) collectRecords(ctx context.Context, task *ExtractTask) (
171182
digest: record.digest,
172183
planDigest: record.planDigest,
173184
}
185+
record.userName = row.GetString(6)
174186
record.tables = make([]tableNamePair, 0)
175187
setRecord, err := w.handleTableNames(tableNames, record)
176188
if err != nil {
@@ -324,7 +336,7 @@ func (w *extractWorker) decodeBinaryPlan(ctx context.Context, bPlan string) (str
324336
| |-digest1.sql
325337
| |-...
326338
*/
327-
func (w *extractWorker) dumpExtractPlanPackage(p *extractPlanPackage) (name string, err error) {
339+
func (w *extractWorker) dumpExtractPlanPackage(task *ExtractTask, p *extractPlanPackage) (name string, err error) {
328340
f, name, err := GenerateExtractFile()
329341
if err != nil {
330342
return "", err
@@ -351,7 +363,7 @@ func (w *extractWorker) dumpExtractPlanPackage(p *extractPlanPackage) (name stri
351363
return "", err
352364
}
353365
// dump extract plan task meta
354-
if err = dumpExtractMeta(ExtractPlanType, zw); err != nil {
366+
if err = dumpExtractMeta(task, zw); err != nil {
355367
return "", err
356368
}
357369
// Dump Schema and View
@@ -371,8 +383,10 @@ func (w *extractWorker) dumpExtractPlanPackage(p *extractPlanPackage) (name stri
371383
return "", err
372384
}
373385
// Dump stats
374-
if err = dumpStats(zw, p.tables, GetDomain(w.sctx)); err != nil {
375-
return "", err
386+
if !task.SkipStats {
387+
if err = dumpStats(zw, p.tables, GetDomain(w.sctx)); err != nil {
388+
return "", err
389+
}
376390
}
377391
// Dump sqls and plan
378392
if err = dumpSQLRecords(p.records, zw); err != nil {
@@ -404,6 +418,7 @@ type singleSQLRecord struct {
404418
SQL string `json:"sql"`
405419
Digest string `json:"digest"`
406420
BinaryPlan string `json:"binaryPlan"`
421+
UserName string `json:"userName"`
407422
}
408423

409424
// dumpSQLRecord dumps sql records into one file for each record, the format is in json.
@@ -418,6 +433,7 @@ func dumpSQLRecord(record *stmtSummaryHistoryRecord, path string, zw *zip.Writer
418433
SQL: record.sql,
419434
Digest: record.digest,
420435
BinaryPlan: record.binaryPlan,
436+
UserName: record.userName,
421437
}
422438
content, err := json.Marshal(singleSQLRecord)
423439
if err != nil {
@@ -430,13 +446,18 @@ func dumpSQLRecord(record *stmtSummaryHistoryRecord, path string, zw *zip.Writer
430446
return nil
431447
}
432448

433-
func dumpExtractMeta(t ExtractType, zw *zip.Writer) error {
449+
func dumpExtractMeta(task *ExtractTask, zw *zip.Writer) error {
434450
cf, err := zw.Create(ExtractMetaFile)
435451
if err != nil {
436452
return errors.AddStack(err)
437453
}
438454
varMap := make(map[string]string)
439-
varMap[ExtractTaskType] = taskTypeToString(t)
455+
varMap[ExtractTaskType] = taskTypeToString(task.ExtractType)
456+
switch task.ExtractType {
457+
case ExtractPlanType:
458+
varMap[ExtractPlanTaskSkipStats] = strconv.FormatBool(task.SkipStats)
459+
}
460+
440461
if err := toml.NewEncoder(cf).Encode(varMap); err != nil {
441462
return errors.AddStack(err)
442463
}
@@ -461,6 +482,7 @@ type stmtSummaryHistoryRecord struct {
461482
planDigest string
462483
sql string
463484
binaryPlan string
485+
userName string
464486

465487
plan string
466488
skip bool

‎domain/extract_test.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,23 @@ import (
2929
"github.com/stretchr/testify/require"
3030
)
3131

32+
func TestExtractPlanWithoutHistoryView(t *testing.T) {
33+
_, dom := testkit.CreateMockStoreAndDomain(t)
34+
extractHandler := dom.GetExtractHandle()
35+
task := domain.NewExtractPlanTask(time.Now(), time.Now())
36+
task.UseHistoryView = false
37+
_, err := extractHandler.ExtractTask(context.Background(), task)
38+
require.NoError(t, err)
39+
}
40+
3241
func TestExtractWithoutStmtSummaryPersistedEnabled(t *testing.T) {
3342
setupStmtSummary()
3443
closeStmtSummary()
3544
_, dom := testkit.CreateMockStoreAndDomain(t)
3645
extractHandler := dom.GetExtractHandle()
37-
_, err := extractHandler.ExtractTask(context.Background(), domain.NewExtractPlanTask(time.Now(), time.Now()))
46+
task := domain.NewExtractPlanTask(time.Now(), time.Now())
47+
task.UseHistoryView = true
48+
_, err := extractHandler.ExtractTask(context.Background(), task)
3849
require.Error(t, err)
3950
}
4051

@@ -61,7 +72,9 @@ func TestExtractHandlePlanTask(t *testing.T) {
6172
time.Sleep(time.Second)
6273
end := time.Now()
6374
extractHandler := dom.GetExtractHandle()
64-
name, err := extractHandler.ExtractTask(context.Background(), domain.NewExtractPlanTask(startTime, end))
75+
task := domain.NewExtractPlanTask(startTime, end)
76+
task.UseHistoryView = true
77+
name, err := extractHandler.ExtractTask(context.Background(), task)
6578
require.NoError(t, err)
6679
require.True(t, len(name) > 0)
6780
}

‎domain/plan_replayer_test.go

+46-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func TestPlanReplayerGC(t *testing.T) {
4646
require.True(t, os.IsNotExist(err))
4747
}
4848

49-
func TestPlanReplayerParseTime(t *testing.T) {
49+
func TestDumpGCFileParseTime(t *testing.T) {
5050
nowTime := time.Now()
5151
name1 := fmt.Sprintf("replayer_single_xxxxxx_%v.zip", nowTime.UnixNano())
5252
pt, err := parseTime(name1)
@@ -60,4 +60,49 @@ func TestPlanReplayerParseTime(t *testing.T) {
6060
name3 := fmt.Sprintf("replayer_single_xxxxxx_%v._zip", nowTime.UnixNano())
6161
_, err = parseTime(name3)
6262
require.NotNil(t, err)
63+
64+
name4 := "extract_-brq6zKMarD9ayaifkHc4A==_1678168728477502000.zip"
65+
_, err = parseTime(name4)
66+
require.NoError(t, err)
67+
68+
var pName string
69+
pName, err = replayer.GeneratePlanReplayerFileName(false, false, false)
70+
require.NoError(t, err)
71+
_, err = parseTime(pName)
72+
require.NoError(t, err)
73+
74+
pName, err = replayer.GeneratePlanReplayerFileName(true, false, false)
75+
require.NoError(t, err)
76+
_, err = parseTime(pName)
77+
require.NoError(t, err)
78+
79+
pName, err = replayer.GeneratePlanReplayerFileName(false, true, false)
80+
require.NoError(t, err)
81+
_, err = parseTime(pName)
82+
require.NoError(t, err)
83+
84+
pName, err = replayer.GeneratePlanReplayerFileName(true, true, false)
85+
require.NoError(t, err)
86+
_, err = parseTime(pName)
87+
require.NoError(t, err)
88+
89+
pName, err = replayer.GeneratePlanReplayerFileName(false, false, true)
90+
require.NoError(t, err)
91+
_, err = parseTime(pName)
92+
require.NoError(t, err)
93+
94+
pName, err = replayer.GeneratePlanReplayerFileName(true, false, true)
95+
require.NoError(t, err)
96+
_, err = parseTime(pName)
97+
require.NoError(t, err)
98+
99+
pName, err = replayer.GeneratePlanReplayerFileName(false, true, true)
100+
require.NoError(t, err)
101+
_, err = parseTime(pName)
102+
require.NoError(t, err)
103+
104+
pName, err = replayer.GeneratePlanReplayerFileName(true, true, true)
105+
require.NoError(t, err)
106+
_, err = parseTime(pName)
107+
require.NoError(t, err)
63108
}

‎server/extract.go

+16-5
Original file line numberDiff line numberDiff line change
@@ -148,15 +148,26 @@ func buildExtractPlanTask(req *http.Request) (*domain.ExtractTask, bool, error)
148148
return nil, false, err
149149
}
150150
}
151-
isDumpStr := req.URL.Query().Get(pIsDump)
152-
isDump, err := strconv.ParseBool(isDumpStr)
153-
if err != nil {
154-
isDump = false
155-
}
151+
isDump := extractBoolParam(pIsDump, false, req)
152+
156153
return &domain.ExtractTask{
157154
ExtractType: domain.ExtractPlanType,
158155
IsBackgroundJob: false,
159156
Begin: begin,
160157
End: end,
158+
SkipStats: extractBoolParam(pIsSkipStats, false, req),
159+
UseHistoryView: extractBoolParam(pIsHistoryView, true, req),
161160
}, isDump, nil
162161
}
162+
163+
func extractBoolParam(param string, defaultValue bool, req *http.Request) bool {
164+
str := req.URL.Query().Get(param)
165+
if len(str) < 1 {
166+
return defaultValue
167+
}
168+
v, err := strconv.ParseBool(str)
169+
if err != nil {
170+
return defaultValue
171+
}
172+
return v
173+
}

‎server/http_handler.go

+10-2
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,16 @@ const (
9191
pDumpPartitionStats = "dumpPartitionStats"
9292
pBegin = "begin"
9393
pEnd = "end"
94-
pType = "type"
95-
pIsDump = "isDump"
94+
)
95+
96+
// For extract task handler
97+
const (
98+
pType = "type"
99+
pIsDump = "isDump"
100+
101+
// For extract plan task handler
102+
pIsSkipStats = "isSkipStats"
103+
pIsHistoryView = "isHistoryView"
96104
)
97105

98106
// For query string

‎util/replayer/replayer.go

+5
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ func GeneratePlanReplayerFile(isCapture, isContinuesCapture, enableHistoricalSta
5050
return zf, fileName, err
5151
}
5252

53+
// GeneratePlanReplayerFileName generates plan replayer capture task name
54+
func GeneratePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture bool) (string, error) {
55+
return generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture)
56+
}
57+
5358
func generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture bool) (string, error) {
5459
// Generate key and create zip file
5560
time := time.Now().UnixNano()

0 commit comments

Comments
 (0)