Skip to content

Commit d66a5bd

Browse files
Hor911pavelvelikhov
Hor911
and
pavelvelikhov
authored
Use GraceCode for Map by default (#8169)
Co-authored-by: Pavel Velikhov <pavelvelikhov@ydb.tech>
1 parent e1efde6 commit d66a5bd

File tree

52 files changed

+1295
-1667
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1295
-1667
lines changed

‎ydb/core/kqp/kqp_default_settings.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,8 @@ DefaultSettings {
7272
Name: "_KqpDisableLlvmForUdfStages"
7373
Value: "false"
7474
}
75+
76+
DefaultSettings {
77+
Name: "UseGraceJoinCoreForMap"
78+
Value: "true"
79+
}

‎ydb/core/kqp/opt/kqp_query_plan.cpp‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1446,7 +1446,13 @@ class TxPlanSerializer {
14461446
}
14471447

14481448
std::variant<ui32, TArgContext> Visit(const TCoFlatMapBase& flatMap, const TCoGraceJoinCore& join, TQueryPlanNode& planNode) {
1449-
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (Grace)";
1449+
auto joinAlgo = "(Grace)";
1450+
for (size_t i=0; i<join.Flags().Size(); i++) {
1451+
if (join.Flags().Item(i).StringValue() == "Broadcast") {
1452+
joinAlgo = "(MapJoin)";
1453+
}
1454+
}
1455+
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join " << joinAlgo;
14501456

14511457
TOperator op;
14521458
op.Properties["Name"] = name;
@@ -1462,7 +1468,13 @@ class TxPlanSerializer {
14621468
}
14631469

14641470
std::variant<ui32, TArgContext> Visit(const TCoGraceJoinCore& join, TQueryPlanNode& planNode) {
1465-
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (Grace)";
1471+
auto joinAlgo = "(Grace)";
1472+
for (size_t i=0; i<join.Flags().Size(); i++) {
1473+
if (join.Flags().Item(i).StringValue() == "Broadcast") {
1474+
joinAlgo = "(MapJoin)";
1475+
}
1476+
}
1477+
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join " << joinAlgo;
14661478

14671479
TOperator op;
14681480
op.Properties["Name"] = name;

‎ydb/core/kqp/ut/common/kqp_ut_common.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,6 +908,11 @@ static void FillPlan(const NYdb::NTable::TScanQueryPart& streamPart, TCollectedS
908908
if (!plan.empty()) {
909909
res.PlanJson = plan;
910910
}
911+
912+
auto ast = res.QueryStats->query_ast();
913+
if (!ast.empty()) {
914+
res.Ast = ast;
915+
}
911916
}
912917
}
913918

@@ -919,6 +924,11 @@ static void FillPlan(const NYdb::NScripting::TYqlResultPart& streamPart, TCollec
919924
if (!plan.empty()) {
920925
res.PlanJson = plan;
921926
}
927+
928+
auto ast = res.QueryStats->query_ast();
929+
if (!ast.empty()) {
930+
res.Ast = ast;
931+
}
922932
}
923933
}
924934

@@ -930,6 +940,11 @@ static void FillPlan(const NYdb::NQuery::TExecuteQueryPart& streamPart, TCollect
930940
if (!plan.empty()) {
931941
res.PlanJson = plan;
932942
}
943+
944+
auto ast = res.QueryStats->query_ast();
945+
if (!ast.empty()) {
946+
res.Ast = ast;
947+
}
933948
}
934949
}
935950

‎ydb/core/kqp/ut/common/kqp_ut_common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ inline TKikimrRunner DefaultKikimrRunner(TVector<NKikimrKqp::TKqpSetting> kqpSet
224224
struct TCollectedStreamResult {
225225
TString ResultSetYson;
226226
TMaybe<TString> PlanJson;
227+
TMaybe<TString> Ast;
227228
TMaybe<Ydb::TableStats::QueryStats> QueryStats;
228229
ui64 RowsCount = 0;
229230
ui64 ConsumedRuFromHeader = 0;

‎ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,8 @@ Y_UNIT_TEST_TWIN(JoinWithSubquery, StreamLookup) {
224224
ON l.Fk = r.Key
225225
);
226226
SELECT j.lValue AS Value FROM $join AS j INNER JOIN `/Root/Kv` AS kv
227-
ON j.lKey = kv.Key;
227+
ON j.lKey = kv.Key
228+
ORDER BY j.lValue;
228229
)";
229230

230231
NKikimrConfig::TAppConfig appConfig;

‎ydb/core/kqp/ut/join/kqp_join_order_ut.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -685,4 +685,4 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
685685

686686
}
687687
}
688-
}
688+
}

‎ydb/core/kqp/ut/join/kqp_join_ut.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
692692
DECLARE $in AS List<Struct<v: String?>>;
693693
SELECT *
694694
FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_1` AS t ON k.v = t.Value
695+
ORDER BY Key
695696
)");
696697

697698
auto params = TParamsBuilder().AddParam("$in").BeginList()

‎ydb/core/kqp/ut/query/kqp_explain_ut.cpp‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,13 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
7979
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
8080
UNIT_ASSERT(res.PlanJson);
8181

82-
Cerr << *res.PlanJson;
82+
Cerr << *res.PlanJson << Endl;
8383

8484
NJson::TJsonValue plan;
8585
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
8686
UNIT_ASSERT(ValidatePlanNodeIds(plan));
8787

88-
auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan");
88+
auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan-Filter");
8989
UNIT_ASSERT(join.IsDefined());
9090
auto left = FindPlanNodeByKv(join, "Table", "EightShard");
9191
UNIT_ASSERT(left.IsDefined());
@@ -106,13 +106,14 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
106106
auto res = CollectStreamResult(it);
107107
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
108108
UNIT_ASSERT(res.PlanJson);
109-
Cerr << *res.PlanJson;
109+
110+
Cerr << *res.PlanJson << Endl;
110111

111112
NJson::TJsonValue plan;
112113
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
113114
UNIT_ASSERT(ValidatePlanNodeIds(plan));
114115

115-
auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan");
116+
auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan-Filter");
116117
UNIT_ASSERT(join.IsDefined());
117118
auto left = FindPlanNodeByKv(join, "Table", "EightShard");
118119
UNIT_ASSERT(left.IsDefined());
@@ -193,7 +194,8 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
193194
auto res = CollectStreamResult(it);
194195
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
195196
UNIT_ASSERT(res.PlanJson);
196-
Cerr << *res.PlanJson;
197+
198+
Cout << *res.PlanJson << Endl;
197199

198200
NJson::TJsonValue plan;
199201
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
@@ -202,7 +204,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
202204
auto join = FindPlanNodeByKv(
203205
plan,
204206
"Node Type",
205-
"Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"
207+
"Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan-Filter"
206208
);
207209

208210
UNIT_ASSERT(join.IsDefined());
@@ -365,9 +367,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
365367
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
366368
UNIT_ASSERT(ValidatePlanNodeIds(plan));
367369

368-
auto join1 = FindPlanNodeByKv(plan, "Node Type", "Sort-InnerJoin (MapJoin)-Filter");
370+
auto join1 = FindPlanNodeByKv(plan, "Node Type", "Sort-InnerJoin (MapJoin)-Filter-Filter");
369371
UNIT_ASSERT(join1.IsDefined());
370-
auto join2 = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter");
372+
auto join2 = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-Filter");
371373
UNIT_ASSERT(join2.IsDefined());
372374
}
373375

@@ -906,6 +908,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
906908
)").ExtractValueSync();
907909
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
908910

911+
Cerr << result.GetPlan() << Endl;
912+
Cout << result.GetPlan() << Endl;
913+
909914
NJson::TJsonValue plan;
910915
NJson::ReadJsonTree(result.GetPlan(), &plan, true);
911916

@@ -918,8 +923,8 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
918923

919924
auto cteLink1 = FindPlanNodeByKv(
920925
plan,
921-
"CTE Name",
922-
"precompute_1_0"
926+
"Subplan Name",
927+
"CTE precompute_1_0"
923928
);
924929

925930
UNIT_ASSERT(cteLink1.IsDefined());

‎ydb/library/yql/core/type_ann/type_ann_join.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -617,7 +617,7 @@ namespace NTypeAnnImpl {
617617
}
618618

619619
for (auto i = 0U; i < input->Tail().ChildrenSize(); ++i) {
620-
if (const auto& flag = *input->Tail().Child(i); !flag.IsAtom({"LeftAny", "RightAny"})) {
620+
if (const auto& flag = *input->Tail().Child(i); !flag.IsAtom({"LeftAny", "RightAny", "Broadcast"})) {
621621
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(flag.Pos()), TStringBuilder() << "Unsupported grace join option: " << flag.Content()));
622622
return IGraphTransformer::TStatus::Error;
623623
}

‎ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,10 @@
5050
{
5151
"Name": "TDqPhyGraceJoin",
5252
"Base": "TDqJoinBase",
53-
"Match": {"Type": "Callable", "Name": "DqPhyGraceJoin"}
53+
"Match": {"Type": "Callable", "Name": "DqPhyGraceJoin"},
54+
"Children": [
55+
{"Index": 8, "Name": "Flags", "Type": "TCoAtomList", "Optional": true}
56+
]
5457
},
5558
{
5659
"Name": "TDqPhyMapJoin",

0 commit comments

Comments
 (0)