Skip to content

Commit 88174d2

Browse files
solotzgsolotzg
authored
planner: support data compression in Broadcast / Passthrough exchange operator; optimize process about choosing Broadcast Join; (#41968)
ref #40494
1 parent b96c55d commit 88174d2

18 files changed

+815
-332
lines changed

‎executor/tiflashtest/tiflash_test.go

+57
Original file line numberDiff line numberDiff line change
@@ -1649,3 +1649,60 @@ func TestDisaggregatedTiFlashGeneratedColumn(t *testing.T) {
16491649
test1(false)
16501650
test2()
16511651
}
1652+
1653+
func TestMppStoreCntWithErrors(t *testing.T) {
1654+
// mock non-root tasks return error
1655+
var mppStoreCountPDError = "github.com/pingcap/tidb/store/copr/mppStoreCountPDError"
1656+
var mppStoreCountSetMPPCnt = "github.com/pingcap/tidb/store/copr/mppStoreCountSetMPPCnt"
1657+
var mppStoreCountSetLastUpdateTime = "github.com/pingcap/tidb/store/copr/mppStoreCountSetLastUpdateTime"
1658+
var mppStoreCountSetLastUpdateTimeP2 = "github.com/pingcap/tidb/store/copr/mppStoreCountSetLastUpdateTimeP2"
1659+
1660+
store := testkit.CreateMockStore(t, withMockTiFlash(3))
1661+
{
1662+
mppCnt, err := store.GetMPPClient().GetMPPStoreCount()
1663+
require.Nil(t, err)
1664+
require.Equal(t, mppCnt, 3)
1665+
}
1666+
require.Nil(t, failpoint.Enable(mppStoreCountSetMPPCnt, `return(1000)`))
1667+
{
1668+
mppCnt, err := store.GetMPPClient().GetMPPStoreCount()
1669+
require.Nil(t, err)
1670+
// meet cache
1671+
require.Equal(t, mppCnt, 3)
1672+
}
1673+
require.Nil(t, failpoint.Enable(mppStoreCountSetLastUpdateTime, `return("0")`))
1674+
{
1675+
mppCnt, err := store.GetMPPClient().GetMPPStoreCount()
1676+
require.Nil(t, err)
1677+
// update cache
1678+
require.Equal(t, mppCnt, 1000)
1679+
}
1680+
require.Nil(t, failpoint.Enable(mppStoreCountPDError, `return(true)`))
1681+
{
1682+
_, err := store.GetMPPClient().GetMPPStoreCount()
1683+
require.Error(t, err)
1684+
}
1685+
require.Nil(t, failpoint.Disable(mppStoreCountPDError))
1686+
require.Nil(t, failpoint.Enable(mppStoreCountSetMPPCnt, `return(2222)`))
1687+
// set last update time to the latest
1688+
require.Nil(t, failpoint.Enable(mppStoreCountSetLastUpdateTime, fmt.Sprintf(`return("%d")`, time.Now().UnixMicro())))
1689+
{
1690+
mppCnt, err := store.GetMPPClient().GetMPPStoreCount()
1691+
require.Nil(t, err)
1692+
// still update cache
1693+
require.Equal(t, mppCnt, 2222)
1694+
}
1695+
require.Nil(t, failpoint.Enable(mppStoreCountSetLastUpdateTime, `return("1")`))
1696+
// fail to get lock and old cache
1697+
require.Nil(t, failpoint.Enable(mppStoreCountSetLastUpdateTimeP2, `return("2")`))
1698+
require.Nil(t, failpoint.Enable(mppStoreCountPDError, `return(true)`))
1699+
{
1700+
mppCnt, err := store.GetMPPClient().GetMPPStoreCount()
1701+
require.Nil(t, err)
1702+
require.Equal(t, mppCnt, 2222)
1703+
}
1704+
require.Nil(t, failpoint.Disable(mppStoreCountSetMPPCnt))
1705+
require.Nil(t, failpoint.Disable(mppStoreCountSetLastUpdateTime))
1706+
require.Nil(t, failpoint.Disable(mppStoreCountSetLastUpdateTimeP2))
1707+
require.Nil(t, failpoint.Disable(mppStoreCountPDError))
1708+
}

‎kv/mpp.go

+2
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ type MPPClient interface {
154154
ConstructMPPTasks(context.Context, *MPPBuildTasksRequest, time.Duration, tiflashcompute.DispatchPolicy) ([]MPPTaskMeta, error)
155155
// DispatchMPPTasks dispatches ALL mpp requests at once, and returns an iterator that transfers the data.
156156
DispatchMPPTasks(ctx context.Context, vars interface{}, reqs []*MPPDispatchRequest, needTriggerFallback bool, startTs uint64, mppQueryID MPPQueryID, mppVersion MppVersion, memTracker *memory.Tracker) Response
157+
// GetMPPStoreCount returns number of TiFlash stores if there is no error, else return (0, error)
158+
GetMPPStoreCount() (int, error)
157159
}
158160

159161
// MPPBuildTasksRequest request the stores allocation for a mpp plan fragment.

‎planner/core/casetest/physical_plan_test.go

+115
Original file line numberDiff line numberDiff line change
@@ -770,6 +770,121 @@ func TestMPPHintsScope(t *testing.T) {
770770
}
771771
}
772772

773+
func TestMPPBCJModel(t *testing.T) {
774+
/*
775+
if there are 3 mpp stores, planner won't choose broadcast join enven if `tidb_prefer_broadcast_join_by_exchange_data_size` is ON
776+
broadcast exchange size:
777+
Build: 2 * sizeof(Data)
778+
Probe: 0
779+
exchange size: Build = 2 * sizeof(Data)
780+
hash exchange size:
781+
Build: sizeof(Data) * 2 / 3
782+
Probe: sizeof(Data) * 2 / 3
783+
exchange size: Build + Probe = 4/3 * sizeof(Data)
784+
*/
785+
store := testkit.CreateMockStore(t, internal.WithMockTiFlash(3))
786+
{
787+
cnt, err := store.GetMPPClient().GetMPPStoreCount()
788+
require.Equal(t, cnt, 3)
789+
require.Nil(t, err)
790+
}
791+
tk := testkit.NewTestKit(t, store)
792+
tk.MustExec("use test")
793+
tk.MustExec("create table t (a int, b int, c int, index idx_a(a), index idx_b(b))")
794+
tk.MustExec("alter table t set tiflash replica 1")
795+
tb := external.GetTableByName(t, tk, "test", "t")
796+
err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
797+
require.NoError(t, err)
798+
799+
var input []string
800+
var output []struct {
801+
SQL string
802+
Plan []string
803+
Warn []string
804+
}
805+
planSuiteData := GetPlanSuiteData()
806+
planSuiteData.LoadTestCases(t, &input, &output)
807+
for i, tt := range input {
808+
testdata.OnRecord(func() {
809+
output[i].SQL = tt
810+
})
811+
if strings.HasPrefix(tt, "set") || strings.HasPrefix(tt, "UPDATE") {
812+
tk.MustExec(tt)
813+
continue
814+
}
815+
testdata.OnRecord(func() {
816+
output[i].SQL = tt
817+
output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
818+
output[i].Warn = testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings())
819+
})
820+
res := tk.MustQuery(tt)
821+
res.Check(testkit.Rows(output[i].Plan...))
822+
require.Equal(t, output[i].Warn, testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings()))
823+
}
824+
}
825+
826+
func TestMPPBCJModelOneTiFlash(t *testing.T) {
827+
/*
828+
if there are 1 mpp stores, planner should choose broadcast join if `tidb_prefer_broadcast_join_by_exchange_data_size` is ON
829+
broadcast exchange size:
830+
Build: 0 * sizeof(Data)
831+
Probe: 0
832+
exchange size: Build = 0 * sizeof(Data)
833+
hash exchange size:
834+
Build: sizeof(Data) * 0 / 1
835+
Probe: sizeof(Data) * 0 / 1
836+
exchange size: Build + Probe = 0 * sizeof(Data)
837+
*/
838+
store := testkit.CreateMockStore(t, internal.WithMockTiFlash(1))
839+
tk := testkit.NewTestKit(t, store)
840+
tk.MustExec("use test")
841+
tk.MustExec("create table t (a int, b int, c int, index idx_a(a), index idx_b(b))")
842+
tk.MustExec("alter table t set tiflash replica 1")
843+
tb := external.GetTableByName(t, tk, "test", "t")
844+
err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
845+
require.NoError(t, err)
846+
{
847+
cnt, err := store.GetMPPClient().GetMPPStoreCount()
848+
require.Equal(t, cnt, 1)
849+
require.Nil(t, err)
850+
}
851+
{
852+
tk.MustExecToErr("set @@session.tidb_prefer_broadcast_join_by_exchange_data_size=-1")
853+
tk.MustExecToErr("set @@session.tidb_prefer_broadcast_join_by_exchange_data_size=2")
854+
}
855+
{
856+
// no BCJ if `tidb_prefer_broadcast_join_by_exchange_data_size` is OFF
857+
tk.MustExec("set @@session.tidb_broadcast_join_threshold_size=0")
858+
tk.MustExec("set @@session.tidb_broadcast_join_threshold_count=0")
859+
}
860+
861+
var input []string
862+
var output []struct {
863+
SQL string
864+
Plan []string
865+
Warn []string
866+
}
867+
planSuiteData := GetPlanSuiteData()
868+
planSuiteData.LoadTestCases(t, &input, &output)
869+
for i, tt := range input {
870+
testdata.OnRecord(func() {
871+
output[i].SQL = tt
872+
})
873+
if strings.HasPrefix(tt, "set") || strings.HasPrefix(tt, "UPDATE") {
874+
tk.MustExec(tt)
875+
continue
876+
}
877+
testdata.OnRecord(func() {
878+
output[i].SQL = tt
879+
output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
880+
output[i].Warn = testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings())
881+
})
882+
res := tk.MustQuery(tt)
883+
res.Check(testkit.Rows(output[i].Plan...))
884+
require.Equal(t, output[i].Warn, testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings()))
885+
}
886+
}
887+
773888
func TestHintScope(t *testing.T) {
774889
store := testkit.CreateMockStore(t)
775890
tk := testkit.NewTestKit(t, store)

‎planner/core/casetest/testdata/derive_topn_from_window_out.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@
188188
" └─Window 10000.00 mpp[tiflash] row_number()->Column#4 over(order by test.t.b rows between current row and current row)",
189189
" └─Sort 10000.00 mpp[tiflash] test.t.b",
190190
" └─ExchangeReceiver 10000.00 mpp[tiflash] ",
191-
" └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: PassThrough",
191+
" └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: PassThrough, Compression: FAST",
192192
" └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo"
193193
]
194194
},

0 commit comments

Comments
 (0)