Skip to content

Commit 02d9cec

Browse files
pashandor789pashandor789
committed
[CBO] Improve transitive closure algorithm, add cycles processing
1 parent 65aa640 commit 02d9cec

22 files changed

+870
-543
lines changed

‎ydb/core/kqp/ut/common/kqp_ut_common.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1468,6 +1468,24 @@ NJson::TJsonValue GetJoinOrder(const TString& deserializedPlan) {
14681468
return GetJoinOrderImpl(optRoot);
14691469
}
14701470

1471+
NJson::TJsonValue GetJoinOrderFromDetailedJoinOrderImpl(const NJson::TJsonValue& opt) {
1472+
if (!opt.GetMapSafe().contains("table")) {
1473+
NJson::TJsonValue res;
1474+
auto args = opt.GetMapSafe().at("args").GetArraySafe();
1475+
for (size_t i = 0; i < args.size(); ++i) {
1476+
res.AppendValue(GetJoinOrderFromDetailedJoinOrderImpl(args[i]));
1477+
}
1478+
return res;
1479+
}
1480+
1481+
return opt.GetMapSafe().at("table");
1482+
}
1483+
1484+
NJson::TJsonValue GetJoinOrderFromDetailedJoinOrder(const TString& deserializedDetailedJoinOrder) {
1485+
NJson::TJsonValue optRoot;
1486+
NJson::ReadJsonTree(deserializedDetailedJoinOrder, &optRoot, true);
1487+
return GetJoinOrderFromDetailedJoinOrderImpl(optRoot);
1488+
}
14711489

14721490
} // namspace NKqp
14731491
} // namespace NKikimr

‎ydb/core/kqp/ut/common/kqp_ut_common.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,5 +381,7 @@ NJson::TJsonValue GetDetailedJoinOrder(const TString& deserializedPlan, const TG
381381
/* Gets tables join order without details : only tables. */
382382
NJson::TJsonValue GetJoinOrder(const TString& deserializedPlan);
383383

384+
NJson::TJsonValue GetJoinOrderFromDetailedJoinOrder(const TString& deserializedDetailedJoinOrder);
385+
384386
} // namespace NKqp
385387
} // namespace NKikimr

‎ydb/core/kqp/ut/join/data/join_order/lookupbug.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,4 @@
3232
}
3333
]
3434
}
35+

‎ydb/core/kqp/ut/join/data/join_order/tpcc.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@
1212
}
1313
]
1414
}
15+

‎ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json

Lines changed: 141 additions & 140 deletions
Large diffs are not rendered by default.

‎ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s_column_store.json

Lines changed: 141 additions & 140 deletions
Large diffs are not rendered by default.

‎ydb/core/kqp/ut/join/data/join_order/tpcds64_small_1000s.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,4 @@
5252
}
5353
]
5454
}
55+

‎ydb/core/kqp/ut/join/data/join_order/tpcds64_small_1000s_column_store.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,4 @@
5252
}
5353
]
5454
}
55+

‎ydb/core/kqp/ut/join/data/join_order/tpcds78_1000s.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,3 +82,4 @@
8282
}
8383
]
8484
}
85+

‎ydb/core/kqp/ut/join/data/join_order/tpcds78_1000s_column_store.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,3 +82,4 @@
8282
}
8383
]
8484
}
85+

‎ydb/core/kqp/ut/join/data/join_order/tpch2_1000s.json

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,39 +7,39 @@
77
"args":
88
[
99
{
10-
"op_name":"TableFullScan",
11-
"table":"partsupp"
12-
},
13-
{
14-
"op_name":"InnerJoin (MapJoin)",
10+
"op_name":"InnerJoin (Grace)",
1511
"args":
1612
[
1713
{
1814
"op_name":"TableFullScan",
19-
"table":"supplier"
15+
"table":"partsupp"
2016
},
2117
{
2218
"op_name":"InnerJoin (MapJoin)",
2319
"args":
2420
[
2521
{
26-
"op_name":"TableLookup",
27-
"table":"region"
22+
"op_name":"TableFullScan",
23+
"table":"supplier"
2824
},
2925
{
30-
"op_name":"TableFullScan",
31-
"table":"nation"
26+
"op_name":"InnerJoin (MapJoin)",
27+
"args":
28+
[
29+
{
30+
"op_name":"TableLookup",
31+
"table":"region"
32+
},
33+
{
34+
"op_name":"TableFullScan",
35+
"table":"nation"
36+
}
37+
]
3238
}
3339
]
3440
}
3541
]
36-
}
37-
]
38-
},
39-
{
40-
"op_name":"InnerJoin (Grace)",
41-
"args":
42-
[
42+
},
4343
{
4444
"op_name":"InnerJoin (Grace)",
4545
"args":
@@ -73,12 +73,13 @@
7373
]
7474
}
7575
]
76-
},
77-
{
78-
"op_name":"TableFullScan",
79-
"table":"part"
8076
}
8177
]
78+
},
79+
{
80+
"op_name":"TableFullScan",
81+
"table":"part"
8282
}
8383
]
8484
}
85+

‎ydb/core/kqp/ut/join/data/join_order/tpch2_1000s_column_store.json

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,39 +7,39 @@
77
"args":
88
[
99
{
10-
"op_name":"TableFullScan",
11-
"table":"partsupp"
12-
},
13-
{
14-
"op_name":"InnerJoin (MapJoin)",
10+
"op_name":"InnerJoin (Grace)",
1511
"args":
1612
[
1713
{
1814
"op_name":"TableFullScan",
19-
"table":"supplier"
15+
"table":"partsupp"
2016
},
2117
{
2218
"op_name":"InnerJoin (MapJoin)",
2319
"args":
2420
[
2521
{
2622
"op_name":"TableFullScan",
27-
"table":"nation"
23+
"table":"supplier"
2824
},
2925
{
30-
"op_name":"TableFullScan",
31-
"table":"region"
26+
"op_name":"InnerJoin (MapJoin)",
27+
"args":
28+
[
29+
{
30+
"op_name":"TableFullScan",
31+
"table":"nation"
32+
},
33+
{
34+
"op_name":"TableFullScan",
35+
"table":"region"
36+
}
37+
]
3238
}
3339
]
3440
}
3541
]
36-
}
37-
]
38-
},
39-
{
40-
"op_name":"InnerJoin (Grace)",
41-
"args":
42-
[
42+
},
4343
{
4444
"op_name":"InnerJoin (Grace)",
4545
"args":
@@ -73,12 +73,13 @@
7373
]
7474
}
7575
]
76-
},
77-
{
78-
"op_name":"TableFullScan",
79-
"table":"part"
8076
}
8177
]
78+
},
79+
{
80+
"op_name":"TableFullScan",
81+
"table":"part"
8282
}
8383
]
8484
}
85+

‎ydb/core/kqp/ut/join/data/join_order/tpch9_1000s.json

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,22 @@
22
"op_name":"InnerJoin (Grace)",
33
"args":
44
[
5+
{
6+
"op_name":"TableFullScan",
7+
"table":"orders"
8+
},
59
{
610
"op_name":"InnerJoin (Grace)",
711
"args":
812
[
913
{
1014
"op_name":"TableFullScan",
11-
"table":"orders"
15+
"table":"lineitem"
1216
},
1317
{
1418
"op_name":"InnerJoin (Grace)",
1519
"args":
1620
[
17-
{
18-
"op_name":"TableFullScan",
19-
"table":"lineitem"
20-
},
2121
{
2222
"op_name":"InnerJoin (Grace)",
2323
"args":
@@ -31,24 +31,25 @@
3131
"table":"part"
3232
}
3333
]
34+
},
35+
{
36+
"op_name":"InnerJoin (MapJoin)",
37+
"args":
38+
[
39+
{
40+
"op_name":"TableFullScan",
41+
"table":"supplier"
42+
},
43+
{
44+
"op_name":"TableFullScan",
45+
"table":"nation"
46+
}
47+
]
3448
}
3549
]
3650
}
3751
]
38-
},
39-
{
40-
"op_name":"InnerJoin (MapJoin)",
41-
"args":
42-
[
43-
{
44-
"op_name":"TableFullScan",
45-
"table":"supplier"
46-
},
47-
{
48-
"op_name":"TableFullScan",
49-
"table":"nation"
50-
}
51-
]
5252
}
5353
]
5454
}
55+

‎ydb/core/kqp/ut/join/data/join_order/tpch9_1000s_column_store.json

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,22 @@
22
"op_name":"InnerJoin (Grace)",
33
"args":
44
[
5+
{
6+
"op_name":"TableFullScan",
7+
"table":"orders"
8+
},
59
{
610
"op_name":"InnerJoin (Grace)",
711
"args":
812
[
913
{
1014
"op_name":"TableFullScan",
11-
"table":"orders"
15+
"table":"lineitem"
1216
},
1317
{
1418
"op_name":"InnerJoin (Grace)",
1519
"args":
1620
[
17-
{
18-
"op_name":"TableFullScan",
19-
"table":"lineitem"
20-
},
2121
{
2222
"op_name":"InnerJoin (Grace)",
2323
"args":
@@ -31,24 +31,25 @@
3131
"table":"part"
3232
}
3333
]
34+
},
35+
{
36+
"op_name":"InnerJoin (MapJoin)",
37+
"args":
38+
[
39+
{
40+
"op_name":"TableFullScan",
41+
"table":"supplier"
42+
},
43+
{
44+
"op_name":"TableFullScan",
45+
"table":"nation"
46+
}
47+
]
3448
}
3549
]
3650
}
3751
]
38-
},
39-
{
40-
"op_name":"InnerJoin (MapJoin)",
41-
"args":
42-
[
43-
{
44-
"op_name":"TableFullScan",
45-
"table":"supplier"
46-
},
47-
{
48-
"op_name":"TableFullScan",
49-
"table":"nation"
50-
}
51-
]
5252
}
5353
]
5454
}
55+

‎ydb/core/kqp/ut/join/kqp_join_order_ut.cpp

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ void CreateTables(TSession session, const TString& schemaPath, bool useColumnSto
5555
res.GetIssues().PrintTo(Cerr);
5656
UNIT_ASSERT(res.IsSuccess());
5757
}
58+
59+
TString GetPrettyJSON(const NJson::TJsonValue& json) {
60+
TStringStream ss;
61+
NJsonWriter::TBuf writer;
62+
writer.SetIndentSpaces(2);
63+
writer.WriteJsonValue(&json);
64+
writer.FlushTo(&ss); ss << Endl;
65+
return ss.Str();
66+
}
67+
5868
/*
5969
* A basic join order test. We define 5 tables sharing the same
6070
* key attribute and construct various full clique join queries
@@ -548,21 +558,19 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
548558
correctJoinOrderPath = correctJoinOrderPath.substr(0, correctJoinOrderPath.find(".json")) + "_column_store.json";
549559
}
550560

551-
auto currentJoinOrder = GetDetailedJoinOrder(result.GetPlan());
552-
Cerr << currentJoinOrder << Endl;
561+
auto currentJoinOrder = GetPrettyJSON(GetDetailedJoinOrder(result.GetPlan()));
562+
553563
/* to canonize the tests use --test-param CANONIZE_JOIN_ORDER_TESTS=TRUE */
554564
TString canonize = GetTestParam("CANONIZE_JOIN_ORDER_TESTS"); canonize.to_lower();
555565
if (canonize.equal("true")) {
556566
Cerr << "--------------------CANONIZING THE TESTS--------------------";
557567
TOFStream stream(SRC_("data/" + correctJoinOrderPath));
558-
NJsonWriter::TBuf writer;
559-
writer.SetIndentSpaces(2);
560-
writer.WriteJsonValue(&currentJoinOrder);
561-
writer.FlushTo(&stream);
562-
stream << Endl;
568+
stream << currentJoinOrder << Endl;
563569
}
564570

565571
TString ref = GetStatic(correctJoinOrderPath);
572+
Cout << "actual\n" << GetJoinOrder(result.GetPlan()).GetStringRobust() << Endl;
573+
Cout << "expected\n" << GetJoinOrderFromDetailedJoinOrder(ref).GetStringRobust() << Endl;
566574
UNIT_ASSERT(JoinOrderAndAlgosMatch(result.GetPlan(), ref));
567575
}
568576
}

0 commit comments

Comments
 (0)