Skip to content

Commit

Permalink
planner: add optimizer trace framework for physicalOptimize (pingcap#…
Browse files Browse the repository at this point in the history 
…30335)

close pingcap#30334
  • Loading branch information
@rebelice
rebelice authored Jan 5, 2022
1 parent 3de7265 commit 9acb2c2
Show file tree
Hide file tree
Showing 7 changed files with 248 additions and 26 deletions.
110 changes: 94 additions & 16 deletions planner/core/find_best_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/tracing"
"go.uber.org/zap"
)

Expand Down Expand Up @@ -136,7 +137,7 @@ func GetPropByOrderByItemsContainScalarFunc(items []*util.ByItems) (*property.Ph
return &property.PhysicalProperty{SortItems: propItems}, true, onlyColumn
}

func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
// If the required property is not empty and the row count > 1,
// we cannot ensure this required property.
// But if the row count is 0 or 1, we don't need to care about the property.
Expand All @@ -151,7 +152,7 @@ func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCou
return &rootTask{p: dual, isEmpty: p.RowCount == 0}, 1, nil
}

func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
if !prop.IsEmpty() || planCounter.Empty() {
return invalidTask, 0, nil
}
Expand All @@ -161,7 +162,7 @@ func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter
return &rootTask{p: pShow}, 1, nil
}

func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
if !prop.IsEmpty() || planCounter.Empty() {
return invalidTask, 0, nil
}
Expand All @@ -172,7 +173,7 @@ func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty, planC
}

// rebuildChildTasks rebuilds the childTasks to make the clock_th combination.
func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan, childCnts []int64, planCounter int64, TS uint64) error {
func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan, childCnts []int64, planCounter int64, TS uint64, opt *physicalOptimizeOp) error {
// The taskMap of children nodes should be rolled back first.
for _, child := range p.children {
child.rollBackTaskMap(TS)
Expand All @@ -187,7 +188,7 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan,
for j, child := range p.children {
multAll /= childCnts[j]
curClock = PlanCounterTp((planCounter-1)/multAll + 1)
childTask, _, err := child.findBestTask(pp.GetChildReqProps(j), &curClock)
childTask, _, err := child.findBestTask(pp.GetChildReqProps(j), &curClock, opt)
planCounter = (planCounter-1)%multAll + 1
if err != nil {
return err
Expand All @@ -203,21 +204,23 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan,
return nil
}

func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp) (task, int64, error) {
func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
var bestTask task = invalidTask
var curCntPlan, cntPlan int64
childTasks := make([]task, 0, len(p.children))
childCnts := make([]int64, len(p.children))
cntPlan = 0
for _, pp := range physicalPlans {
candidateInfo := opt.appendCandidate(p, pp, prop.String())
// Find best child tasks firstly.
childTasks = childTasks[:0]
// The curCntPlan records the number of possible plans for pp
curCntPlan = 1
TimeStampNow := p.GetLogicalTS4TaskMap()
savedPlanID := p.ctx.GetSessionVars().PlanID
for j, child := range p.children {
childTask, cnt, err := child.findBestTask(pp.GetChildReqProps(j), &PlanCounterDisabled)
childProp := pp.GetChildReqProps(j)
childTask, cnt, err := child.findBestTask(childProp, &PlanCounterDisabled, opt)
childCnts[j] = cnt
if err != nil {
return nil, 0, err
Expand All @@ -227,6 +230,9 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
break
}
childTasks = append(childTasks, childTask)
if opt != nil && childTask != nil {
opt.appendChildToCandidate(candidateInfo, childTask.plan())
}
}

// This check makes sure that there is no invalid child task.
Expand All @@ -238,7 +244,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
if planCounter.IsForce() && int64(*planCounter) <= curCntPlan {
p.ctx.GetSessionVars().PlanID = savedPlanID
curCntPlan = int64(*planCounter)
err := p.rebuildChildTasks(&childTasks, pp, childCnts, int64(*planCounter), TimeStampNow)
err := p.rebuildChildTasks(&childTasks, pp, childCnts, int64(*planCounter), TimeStampNow, opt)
if err != nil {
return nil, 0, err
}
Expand Down Expand Up @@ -274,6 +280,9 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
bestTask = curTask
break
}
if candidateInfo != nil {
candidateInfo.SetCost(curTask.cost())
}
// Get the most efficient one.
if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) {
bestTask = curTask
Expand All @@ -282,8 +291,71 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
return bestTask, cntPlan, nil
}

type physicalOptimizeOp struct {
// tracer is goring to track optimize steps during physical optimizing
tracer *tracing.PhysicalOptimizeTracer
}

func defaultPhysicalOptimizeOption() *physicalOptimizeOp {
return &physicalOptimizeOp{}
}

func (op *physicalOptimizeOp) withEnableOptimizeTracer(tracer *tracing.PhysicalOptimizeTracer) *physicalOptimizeOp {
op.tracer = tracer
return op
}

func (op *physicalOptimizeOp) buildPhysicalOptimizeTraceInfo(p LogicalPlan, prop string) *tracing.PhysicalOptimizeTraceInfo {
if op == nil || op.tracer == nil {
return nil
}
name := tracing.CodecPlanName(p.TP(), p.ID())
if _, ok := op.tracer.State[name]; !ok {
op.tracer.State[name] = make(map[string]*tracing.PhysicalOptimizeTraceInfo)
}
if info, ok := op.tracer.State[name][prop]; ok {
return info
}
traceInfo := &tracing.PhysicalOptimizeTraceInfo{Property: prop}
op.tracer.State[name][prop] = traceInfo
return traceInfo
}

func (op *physicalOptimizeOp) appendChildToCandidate(candidateInfo *tracing.PhysicalPlanTrace, plan PhysicalPlan) {
if op == nil || op.tracer == nil || candidateInfo == nil {
return
}
childPhysicalPlanTrace := &tracing.PhysicalPlanTrace{TP: plan.TP(), ID: plan.ID(), Info: plan.ExplainInfo(), Cost: plan.Cost()}
candidateInfo.Children = append(candidateInfo.Children, childPhysicalPlanTrace)
}

func (op *physicalOptimizeOp) setBest(lp LogicalPlan, pp PhysicalPlan, prop string) {
if op == nil || op.tracer == nil {
return
}
traceInfo := op.tracer.State[fmt.Sprintf("%v_%v", lp.TP(), lp.ID())][prop]
if traceInfo == nil {
return
}
traceInfo.BestTask = &tracing.PhysicalPlanTrace{ID: pp.ID(), TP: pp.TP(), Info: pp.ExplainInfo(), Cost: pp.Cost()}
}

func (op *physicalOptimizeOp) appendCandidate(logicalPlan *baseLogicalPlan, physicalPlan PhysicalPlan, prop string) *tracing.PhysicalPlanTrace {
if op == nil || op.tracer == nil {
return nil
}
PhysicalPlanTrace := &tracing.PhysicalPlanTrace{TP: physicalPlan.TP(), ID: physicalPlan.ID(), Info: physicalPlan.ExplainInfo()}
name := tracing.CodecPlanName(logicalPlan.TP(), logicalPlan.ID())
traceInfo := op.tracer.State[name][prop]
if traceInfo == nil {
return nil
}
traceInfo.Candidates = append(traceInfo.Candidates, PhysicalPlanTrace)
return PhysicalPlanTrace
}

// findBestTask implements LogicalPlan interface.
func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (bestTask task, cntPlan int64, err error) {
func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (bestTask task, cntPlan int64, err error) {
// If p is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself,
// and set inner child prop nil, so here we do nothing.
if prop == nil {
Expand Down Expand Up @@ -353,15 +425,18 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun

var cnt int64
var curTask task
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter); err != nil {
if opt != nil {
opt.buildPhysicalOptimizeTraceInfo(p, newProp.String())
}
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter, opt); err != nil {
return nil, 0, err
}
cntPlan += cnt
if planCounter.Empty() {
goto END
}

curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, true, planCounter)
curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, true, planCounter, opt)
if err != nil {
return nil, 0, err
}
Expand All @@ -376,10 +451,13 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun

END:
p.storeTask(prop, bestTask)
if opt != nil {
opt.setBest(p.self, bestTask.plan(), prop.String())
}
return bestTask, cntPlan, nil
}

func (p *LogicalMemTable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (p *LogicalMemTable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
if !prop.IsEmpty() || planCounter.Empty() {
return invalidTask, 0, nil
}
Expand Down Expand Up @@ -689,7 +767,7 @@ func (ds *DataSource) isPointGetConvertableSchema() bool {

// findBestTask implements the PhysicalPlan interface.
// It will enumerate all the available indices and choose a plan with least cost.
func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
// If ds is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself,
// and set inner child prop nil, so here we do nothing.
if prop == nil {
Expand All @@ -710,7 +788,7 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
if prop.CanAddEnforcer {
// First, get the bestTask without enforced prop
prop.CanAddEnforcer = false
t, cnt, err = ds.findBestTask(prop, planCounter)
t, cnt, err = ds.findBestTask(prop, planCounter, opt)
if err != nil {
return nil, 0, err
}
Expand Down Expand Up @@ -2169,7 +2247,7 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper
return is, cost, rowCount
}

func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
if !prop.IsEmpty() && !prop.CanAddEnforcer {
return invalidTask, 1, nil
}
Expand All @@ -2187,7 +2265,7 @@ func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *
return t, 1, nil
}

func (p *LogicalCTETable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (p *LogicalCTETable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
if !prop.IsEmpty() {
return nil, 1, nil
}
Expand Down
16 changes: 8 additions & 8 deletions planner/core/find_best_task_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (ds mockDataSource) Init(ctx sessionctx.Context) *mockDataSource {
return &ds
}

func (ds *mockDataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (ds *mockDataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
// It can satisfy any of the property!
// Just use a TableDual for convenience.
p := PhysicalTableDual{}.Init(ds.ctx, &property.StatsInfo{RowCount: 1}, 0)
Expand Down Expand Up @@ -144,7 +144,7 @@ func TestCostOverflow(t *testing.T) {
mockPlan.SetChildren(mockDS)
// An empty property is enough for this test.
prop := property.NewPhysicalProperty(property.RootTaskType, nil, false, 0, false)
task, _, err := mockPlan.findBestTask(prop, &PlanCounterDisabled)
task, _, err := mockPlan.findBestTask(prop, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
// The cost should be overflowed, but the task shouldn't be invalid.
require.False(t, task.invalid())
Expand All @@ -171,7 +171,7 @@ func TestEnforcedProperty(t *testing.T) {
CanAddEnforcer: false,
}
// should return invalid task because no physical plan can match this property.
task, _, err := mockPlan.findBestTask(prop0, &PlanCounterDisabled)
task, _, err := mockPlan.findBestTask(prop0, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.True(t, task.invalid())

Expand All @@ -180,7 +180,7 @@ func TestEnforcedProperty(t *testing.T) {
CanAddEnforcer: true,
}
// should return the valid task when the property is enforced.
task, _, err = mockPlan.findBestTask(prop1, &PlanCounterDisabled)
task, _, err = mockPlan.findBestTask(prop1, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
}
Expand All @@ -203,7 +203,7 @@ func TestHintCannotFitProperty(t *testing.T) {
SortItems: items,
CanAddEnforcer: true,
}
task, _, err := mockPlan0.findBestTask(prop0, &PlanCounterDisabled)
task, _, err := mockPlan0.findBestTask(prop0, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
_, enforcedSort := task.plan().(*PhysicalSort)
Expand All @@ -219,7 +219,7 @@ func TestHintCannotFitProperty(t *testing.T) {
SortItems: items,
CanAddEnforcer: false,
}
task, _, err = mockPlan0.findBestTask(prop1, &PlanCounterDisabled)
task, _, err = mockPlan0.findBestTask(prop1, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
_, enforcedSort = task.plan().(*PhysicalSort)
Expand All @@ -240,7 +240,7 @@ func TestHintCannotFitProperty(t *testing.T) {
canGeneratePlan2: false,
}.Init(ctx)
mockPlan1.SetChildren(mockDS)
task, _, err = mockPlan1.findBestTask(prop2, &PlanCounterDisabled)
task, _, err = mockPlan1.findBestTask(prop2, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
require.Equal(t, uint16(1), ctx.GetSessionVars().StmtCtx.WarningCount())
Expand All @@ -256,7 +256,7 @@ func TestHintCannotFitProperty(t *testing.T) {
SortItems: items,
CanAddEnforcer: true,
}
task, _, err = mockPlan1.findBestTask(prop3, &PlanCounterDisabled)
task, _, err = mockPlan1.findBestTask(prop3, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
require.Equal(t, uint16(1), ctx.GetSessionVars().StmtCtx.WarningCount())
Expand Down
12 changes: 11 additions & 1 deletion planner/core/optimizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,18 @@ func physicalOptimize(logic LogicalPlan, planCounter *PlanCounterTp) (PhysicalPl
ExpectedCnt: math.MaxFloat64,
}

opt := defaultPhysicalOptimizeOption()
stmtCtx := logic.SCtx().GetSessionVars().StmtCtx
if stmtCtx.EnableOptimizeTrace {
tracer := &tracing.PhysicalOptimizeTracer{State: make(map[string]map[string]*tracing.PhysicalOptimizeTraceInfo)}
opt = opt.withEnableOptimizeTracer(tracer)
defer func() {
stmtCtx.PhysicalOptimizeTrace = tracer
}()
}

logic.SCtx().GetSessionVars().StmtCtx.TaskMapBakTS = 0
t, _, err := logic.findBestTask(prop, planCounter)
t, _, err := logic.findBestTask(prop, planCounter, opt)
if err != nil {
return nil, 0, err
}
Expand Down
Loading

0 comments on commit 9acb2c2

Please sign in to comment.