Skip to content

Commit 811eec6

Browse files
pashandor789pashandor789
authored
Merge 2b26f94 into 6f007f5 
2 parents 6f007f5 + 2b26f94 commit 811eec6

26 files changed

+314
-638
lines changed

‎ydb/core/kqp/executer_actor/kqp_executer_impl.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1652,11 +1652,12 @@ class TKqpExecuterBase : public TActor<TDerived> {
16521652
auto& stage = stageInfo.Meta.GetStage(stageInfo.Id);
16531653

16541654
auto& columnShardHashV1Params = stageInfo.Meta.ColumnShardHashV1Params;
1655-
if (enableShuffleElimination && stageInfo.Meta.ColumnTableInfoPtr) {
1655+
if (enableShuffleElimination && stage.GetIsShuffleEliminated() && stageInfo.Meta.ColumnTableInfoPtr) {
16561656
const auto& tableDesc = stageInfo.Meta.ColumnTableInfoPtr->Description;
16571657
columnShardHashV1Params.SourceShardCount = tableDesc.GetColumnShardCount();
16581658
columnShardHashV1Params.SourceTableKeyColumnTypes = std::make_shared<TVector<NScheme::TTypeInfo>>();
16591659
for (const auto& column: tableDesc.GetSharding().GetHashSharding().GetColumns()) {
1660+
Y_ENSURE(stageInfo.Meta.TableConstInfo->Columns.contains(column), TStringBuilder{} << "Table doesn't have column: " << column);
16601661
auto columnType = stageInfo.Meta.TableConstInfo->Columns.at(column).Type;
16611662
columnShardHashV1Params.SourceTableKeyColumnTypes->push_back(columnType);
16621663
}

‎ydb/core/kqp/opt/logical/kqp_opt_log.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
163163
TMaybeNode<TExprBase> OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) {
164164
auto maxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize);
165165
auto optLevel = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel);
166-
bool enableShuffleElimination = KqpCtx.Config->OptShuffleElimination.Get().GetOrElse(Config->DefaultEnableShuffleElimination);
166+
bool enableShuffleElimination = KqpCtx.Config->OptShuffleElimination.Get().GetOrElse(KqpCtx.Config->DefaultEnableShuffleElimination);
167167
auto providerCtx = TKqpProviderContext(KqpCtx, optLevel);
168168
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPhypDPTableSize, ctx, enableShuffleElimination));
169169
TExprBase output = DqOptimizeEquiJoinWithCosts(node, ctx, TypesCtx, optLevel,

‎ydb/core/kqp/opt/physical/kqp_opt_phy.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ class TKqpPhysicalOptTransformer : public TOptimizeTransformerBase {
452452
bool shuffleEliminationWithMap = KqpCtx.Config->OptShuffleEliminationWithMap.Get().GetOrElse(true);
453453
bool rightCollectStage = !KqpCtx.Config->AllowMultiBroadcasts;
454454
TExprBase output = DqBuildJoin(node, ctx, optCtx, *getParents(), IsGlobal,
455-
pushLeftStage, KqpCtx.Config->GetHashJoinMode(), false, KqpCtx.Config->UseGraceJoinCoreForMap.Get().GetOrElse(false), KqpCtx.Config->OptShuffleElimination.Get().GetOrElse(false), shuffleEliminationWithMap,
455+
pushLeftStage, KqpCtx.Config->GetHashJoinMode(), false, KqpCtx.Config->UseGraceJoinCoreForMap.Get().GetOrElse(false), KqpCtx.Config->OptShuffleElimination.Get().GetOrElse(KqpCtx.Config->DefaultEnableShuffleElimination), shuffleEliminationWithMap,
456456
rightCollectStage
457457
);
458458
DumpAppliedRule("BuildJoin", node.Ptr(), output.Ptr(), ctx);

‎ydb/core/kqp/query_compiler/kqp_query_compiler.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -919,7 +919,7 @@ class TKqpQueryCompiler : public IKqpQueryCompiler {
919919
i.MutableProgram()->MutableSettings()->SetLevelDataPrediction(rPredictor.GetLevelDataVolume(i.GetProgram().GetSettings().GetStageLevel()));
920920
}
921921

922-
txProto.SetEnableShuffleElimination(Config->OptShuffleElimination.Get().GetOrElse(false));
922+
txProto.SetEnableShuffleElimination(Config->OptShuffleElimination.Get().GetOrElse(Config->DefaultEnableShuffleElimination));
923923
txProto.SetHasEffects(hasEffectStage);
924924

925925
for (const auto& paramBinding : tx.ParamBindings()) {
@@ -1336,7 +1336,7 @@ class TKqpQueryCompiler : public IKqpQueryCompiler {
13361336
shuffleProto.AddKeyColumns(TString(keyColumn));
13371337
}
13381338

1339-
if (Config->OptShuffleElimination.Get().GetOrElse(false)) {
1339+
if (Config->OptShuffleElimination.Get().GetOrElse(Config->DefaultEnableShuffleElimination)) {
13401340
auto& columnHashV1 = *shuffleProto.MutableColumnShardHashV1();
13411341

13421342
const auto& outputType = NYql::NDq::GetDqConnectionType(connection, ctx);

‎ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s_column_store.json‎

Lines changed: 56 additions & 164 deletions
Large diffs are not rendered by default.

‎ydb/core/kqp/ut/join/data/join_order/tpcds78_1000s_column_store.json‎

Lines changed: 43 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -3,104 +3,98 @@
33
"args":
44
[
55
{
6-
"op_name":"HashShuffle",
6+
"op_name":"LeftJoin (Grace)",
77
"args":
88
[
99
{
10-
"op_name":"LeftJoin (Grace)",
10+
"op_name":"HashShuffle",
1111
"args":
1212
[
1313
{
1414
"op_name":"HashShuffle",
1515
"args":
1616
[
1717
{
18-
"op_name":"HashShuffle",
18+
"op_name":"LeftJoin (Grace)",
1919
"args":
2020
[
2121
{
22-
"op_name":"LeftJoin (Grace)",
22+
"op_name":"HashShuffle",
2323
"args":
2424
[
2525
{
26-
"op_name":"HashShuffle",
26+
"op_name":"InnerJoin (MapJoin)",
2727
"args":
2828
[
2929
{
30-
"op_name":"InnerJoin (MapJoin)",
31-
"args":
32-
[
33-
{
34-
"op_name":"TableFullScan",
35-
"table":"test\/ds\/store_sales"
36-
},
37-
{
38-
"op_name":"TableFullScan",
39-
"table":"test\/ds\/date_dim"
40-
}
41-
]
42-
}
43-
]
44-
},
45-
{
46-
"op_name":"HashShuffle",
47-
"args":
48-
[
30+
"op_name":"TableFullScan",
31+
"table":"test\/ds\/store_sales"
32+
},
4933
{
5034
"op_name":"TableFullScan",
51-
"table":"test\/ds\/store_returns"
35+
"table":"test\/ds\/date_dim"
5236
}
5337
]
5438
}
5539
]
40+
},
41+
{
42+
"op_name":"HashShuffle",
43+
"args":
44+
[
45+
{
46+
"op_name":"TableFullScan",
47+
"table":"test\/ds\/store_returns"
48+
}
49+
]
5650
}
5751
]
5852
}
5953
]
60-
},
54+
}
55+
]
56+
},
57+
{
58+
"op_name":"HashShuffle",
59+
"args":
60+
[
6161
{
6262
"op_name":"HashShuffle",
6363
"args":
6464
[
6565
{
66-
"op_name":"HashShuffle",
66+
"op_name":"InnerJoin (MapJoin)",
6767
"args":
6868
[
6969
{
70-
"op_name":"InnerJoin (MapJoin)",
70+
"op_name":"LeftJoin (Grace)",
7171
"args":
7272
[
7373
{
74-
"op_name":"LeftJoin (Grace)",
74+
"op_name":"HashShuffle",
7575
"args":
7676
[
7777
{
78-
"op_name":"HashShuffle",
79-
"args":
80-
[
81-
{
82-
"op_name":"TableFullScan",
83-
"table":"test\/ds\/web_sales"
84-
}
85-
]
86-
},
87-
{
88-
"op_name":"HashShuffle",
89-
"args":
90-
[
91-
{
92-
"op_name":"TableFullScan",
93-
"table":"test\/ds\/web_returns"
94-
}
95-
]
78+
"op_name":"TableFullScan",
79+
"table":"test\/ds\/web_sales"
9680
}
9781
]
9882
},
9983
{
100-
"op_name":"TableFullScan",
101-
"table":"test\/ds\/date_dim"
84+
"op_name":"HashShuffle",
85+
"args":
86+
[
87+
{
88+
"op_name":"TableFullScan",
89+
"table":"test\/ds\/web_returns"
90+
}
91+
]
10292
}
10393
]
94+
},
95+
{
96+
"op_name":"TableFullScan",
97+
"table":"test\/ds\/date_dim"
10498
}
10599
]
106100
}

‎ydb/core/kqp/ut/join/data/join_order/tpch10_1000s_column_store.json‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,16 @@
77
"args":
88
[
99
{
10-
"op_name":"HashShuffle",
10+
"op_name":"InnerJoin (MapJoin)",
1111
"args":
1212
[
1313
{
14-
"op_name":"InnerJoin (MapJoin)",
15-
"args":
16-
[
17-
{
18-
"op_name":"TableFullScan",
19-
"table":"customer"
20-
},
21-
{
22-
"op_name":"TableFullScan",
23-
"table":"nation"
24-
}
25-
]
14+
"op_name":"TableFullScan",
15+
"table":"customer"
16+
},
17+
{
18+
"op_name":"TableFullScan",
19+
"table":"nation"
2620
}
2721
]
2822
},
@@ -35,14 +29,8 @@
3529
"args":
3630
[
3731
{
38-
"op_name":"HashShuffle",
39-
"args":
40-
[
41-
{
42-
"op_name":"TableFullScan",
43-
"table":"orders"
44-
}
45-
]
32+
"op_name":"TableFullScan",
33+
"table":"orders"
4634
},
4735
{
4836
"op_name":"HashShuffle",

‎ydb/core/kqp/ut/join/data/join_order/tpch11_1000s_column_store.json‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -21,22 +21,16 @@
2121
]
2222
},
2323
{
24-
"op_name":"HashShuffle",
24+
"op_name":"InnerJoin (MapJoin)",
2525
"args":
2626
[
2727
{
28-
"op_name":"InnerJoin (MapJoin)",
29-
"args":
30-
[
31-
{
32-
"op_name":"TableFullScan",
33-
"table":"supplier"
34-
},
35-
{
36-
"op_name":"TableFullScan",
37-
"table":"nation"
38-
}
39-
]
28+
"op_name":"TableFullScan",
29+
"table":"supplier"
30+
},
31+
{
32+
"op_name":"TableFullScan",
33+
"table":"nation"
4034
}
4135
]
4236
}
@@ -59,22 +53,16 @@
5953
]
6054
},
6155
{
62-
"op_name":"HashShuffle",
56+
"op_name":"InnerJoin (MapJoin)",
6357
"args":
6458
[
6559
{
66-
"op_name":"InnerJoin (MapJoin)",
67-
"args":
68-
[
69-
{
70-
"op_name":"TableFullScan",
71-
"table":"supplier"
72-
},
73-
{
74-
"op_name":"TableFullScan",
75-
"table":"nation"
76-
}
77-
]
60+
"op_name":"TableFullScan",
61+
"table":"supplier"
62+
},
63+
{
64+
"op_name":"TableFullScan",
65+
"table":"nation"
7866
}
7967
]
8068
}

‎ydb/core/kqp/ut/join/data/join_order/tpch13_1000s_column_store.json‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,8 @@
1111
"args":
1212
[
1313
{
14-
"op_name":"HashShuffle",
15-
"args":
16-
[
17-
{
18-
"op_name":"TableFullScan",
19-
"table":"customer"
20-
}
21-
]
14+
"op_name":"TableFullScan",
15+
"table":"customer"
2216
},
2317
{
2418
"op_name":"HashShuffle",

‎ydb/core/kqp/ut/join/data/join_order/tpch14_1000s_column_store.json‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,8 @@
1313
]
1414
},
1515
{
16-
"op_name":"HashShuffle",
17-
"args":
18-
[
19-
{
20-
"op_name":"TableFullScan",
21-
"table":"part"
22-
}
23-
]
16+
"op_name":"TableFullScan",
17+
"table":"part"
2418
}
2519
]
2620
}

0 commit comments

Comments
 (0)