Skip to content

Commit ca105b2

Browse files
MrLolthe1stMrLolthe1st
committed
fixes
1 parent c0dfb76 commit ca105b2

File tree

53 files changed

+454
-250
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+454
-250
lines changed

‎ydb/library/yql/core/common_opt/yql_co_pgselect.cpp‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -928,7 +928,9 @@ TUsedColumns GatherUsedColumns(const TExprNode::TPtr& result, const TExprNode::T
928928

929929
for (ui32 col = 0; col < join->Child(3)->ChildrenSize(); ++col) {
930930
auto lr = join->Child(3)->Child(col);
931-
usedColumns.insert(std::make_pair(TString(lr->Child(0)->Content()), std::make_pair(Max<ui32>(), TString())));
931+
if (lr->Child(0)->IsAtom()) {
932+
usedColumns.insert(std::make_pair(TString(lr->Child(0)->Content()), std::make_pair(Max<ui32>(), TString())));
933+
}
932934
usedColumns.insert(std::make_pair(TString(lr->Child(1)->Content()), std::make_pair(Max<ui32>(), TString())));
933935
usedColumns.erase(TString(join->Child(2)->Child(col)->Content()));
934936
joinUsingColumns.emplace_back(ToString(groupNo), join->Child(2)->Child(col)->Content());
@@ -1530,15 +1532,22 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
15301532
current = cartesian;
15311533
continue;
15321534
}
1533-
if (join->ChildrenSize() > 1 && join->Child(1)->Content() == "using") {
1535+
if (join->ChildrenSize() > 2) {
1536+
Y_ENSURE(join->Child(1)->IsAtom(), "expected only USING clause when join_ops children size > 2");
1537+
Y_ENSURE(join->Child(1)->Content() == "using", "expected only USING clause when join_ops children size > 2");
15341538
Y_ENSURE(join->ChildrenSize() > 3 && join->Child(3)->IsList(), "Excepted list of aliased columns in USING join");
15351539
auto left = current;
15361540
auto right = with;
15371541
TExprNode::TListType leftColumns;
15381542
TExprNode::TListType rightColumns;
1539-
1543+
TExprNode::TListType toRemove;
15401544
for (auto& col: join->Child(3)->ChildrenList()) {
1541-
leftColumns.push_back(col->Child(0));
1545+
if (col->Child(0)->IsAtom()) {
1546+
leftColumns.push_back(col->Child(0));
1547+
} else {
1548+
toRemove.push_back(col->Child(0)->Child(0));
1549+
leftColumns.push_back(col->Child(0)->Child(0));
1550+
}
15421551
rightColumns.push_back(col->Child(1));
15431552
}
15441553
current = BuildEquiJoin(pos, joinType, left, right, leftColumns, rightColumns, ctx);
@@ -1567,6 +1576,11 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
15671576
.Seal()
15681577
.Seal()
15691578
.Build();
1579+
auto removeProjection = ctx.Builder(pos)
1580+
.Lambda()
1581+
.Param("row")
1582+
.Arg(0, "row")
1583+
.Seal().Build();
15701584
current = ctx.Builder(pos)
15711585
.Callable("OrderedMap")
15721586
.Add(0, current)
@@ -1575,7 +1589,17 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
15751589
.Callable("FlattenMembers")
15761590
.List(0)
15771591
.Atom(0, "")
1578-
.Arg(1, "row")
1592+
.Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
1593+
if (toRemove.size()) {
1594+
parent.Callable(1, "RemoveMembers")
1595+
.Arg(0, "row")
1596+
.Add(1, ctx.NewList(pos, std::move(toRemove)))
1597+
.Seal();
1598+
} else {
1599+
parent.Arg(1, "row");
1600+
}
1601+
return parent;
1602+
})
15791603
.Seal()
15801604
.List(1)
15811605
.Atom(0, "")

‎ydb/library/yql/core/type_ann/type_ann_core.cpp‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4708,19 +4708,7 @@ namespace NTypeAnnImpl {
47084708
.Build();
47094709
}
47104710
}
4711-
if (input->ChildrenSize() > 2) {
4712-
output = ctx.Expr.Builder(input->Pos())
4713-
.Callable("CoalesceMembers")
4714-
.Add(0, output)
4715-
.Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
4716-
for (ui32 i = 2; i < input->ChildrenSize(); ++i) {
4717-
parent.Add(i - 1, input->Child(i));
4718-
}
4719-
return parent;
4720-
})
4721-
.Seal()
4722-
.Build();
4723-
}
4711+
47244712
return IGraphTransformer::TStatus::Repeat;
47254713
}
47264714

‎ydb/library/yql/core/type_ann/type_ann_pg.cpp‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2652,7 +2652,7 @@ bool ReplaceProjectionRefs(TExprNode::TPtr& lambda, const TStringBuf& scope, con
26522652
bool ValidateGroups(TInputs& inputs, const THashSet<TString>& possibleAliases,
26532653
const TExprNode& data, TExtContext& ctx, TExprNode::TListType& newGroups, bool& hasNewGroups, bool scanColumnsOnly,
26542654
bool allowAggregates, const TExprNode::TPtr& groupExprs, const TStringBuf& scope,
2655-
const TProjectionOrders* projectionOrders, const TExprNode::TPtr& result) {
2655+
const TProjectionOrders* projectionOrders, const TExprNode::TPtr& result, THashMap<TString, TString> usedInUsing={}) {
26562656
newGroups.clear();
26572657
hasNewGroups = false;
26582658
bool hasColumnRef = false;
@@ -2669,7 +2669,7 @@ bool ValidateGroups(TInputs& inputs, const THashSet<TString>& possibleAliases,
26692669
if (group->Child(0)->IsCallable("Void")) {
26702670
// no effective type yet, scan lambda body
26712671
if (!ScanColumns(group->Tail().TailPtr(), inputs, possibleAliases, nullptr, hasColumnRef,
2672-
refs, &qualifiedRefs, ctx, scanColumnsOnly)) {
2672+
refs, &qualifiedRefs, ctx, scanColumnsOnly, false, usedInUsing)) {
26732673
return false;
26742674
}
26752675

@@ -2701,7 +2701,7 @@ bool ValidateGroups(TInputs& inputs, const THashSet<TString>& possibleAliases,
27012701
}
27022702

27032703
TVector<const TItemExprType*> items;
2704-
AddColumns(inputs, nullptr, refs, &qualifiedRefs, items, ctx.Expr);
2704+
AddColumns(inputs, nullptr, refs, &qualifiedRefs, items, ctx.Expr, usedInUsing);
27052705
auto effectiveType = ctx.Expr.MakeType<TStructExprType>(items);
27062706
if (!effectiveType->Validate(group->Pos(), ctx.Expr)) {
27072707
return false;
@@ -2712,7 +2712,7 @@ bool ValidateGroups(TInputs& inputs, const THashSet<TString>& possibleAliases,
27122712
auto argNode = ctx.Expr.NewArgument(group->Pos(), "row");
27132713
auto arguments = ctx.Expr.NewArguments(group->Pos(), { argNode });
27142714
TExprNode::TPtr newRoot;
2715-
auto status = RebuildLambdaColumns(group->Tail().TailPtr(), argNode, newRoot, inputs, nullptr, ctx);
2715+
auto status = RebuildLambdaColumns(group->Tail().TailPtr(), argNode, newRoot, inputs, nullptr, ctx, usedInUsing);
27162716
if (status == IGraphTransformer::TStatus::Error) {
27172717
return false;
27182718
}
@@ -2799,7 +2799,7 @@ bool IsPlainMemberOverArg(const TExprNode& expr, TStringBuf& memberName) {
27992799
bool ValidateSort(TInputs& inputs, TInputs& subLinkInputs, const THashSet<TString>& possibleAliases,
28002800
const TExprNode& data, TExtContext& ctx, bool& hasNewSort, TExprNode::TListType& newSorts, bool scanColumnsOnly,
28012801
const TExprNode::TPtr& groupExprs, const TStringBuf& scope, const TProjectionOrders* projectionOrders,
2802-
const TExprNode::TPtr& projection) {
2802+
const TExprNode::TPtr& projection, THashMap<TString, TString> usedInUsing={}) {
28032803
newSorts.clear();
28042804
for (ui32 index = 0; index < data.ChildrenSize(); ++index) {
28052805
auto oneSort = data.Child(index);
@@ -2811,7 +2811,7 @@ bool ValidateSort(TInputs& inputs, TInputs& subLinkInputs, const THashSet<TStrin
28112811
THashSet<TString> refs;
28122812
THashMap<TString, THashSet<TString>> qualifiedRefs;
28132813
if (!ScanColumns(oneSort->Child(1)->TailPtr(), inputs, possibleAliases, nullptr, hasColumnRef,
2814-
refs, &qualifiedRefs, ctx, scanColumnsOnly)) {
2814+
refs, &qualifiedRefs, ctx, scanColumnsOnly, false, usedInUsing)) {
28152815
return false;
28162816
}
28172817

@@ -2845,6 +2845,11 @@ bool ValidateSort(TInputs& inputs, TInputs& subLinkInputs, const THashSet<TStrin
28452845

28462846
TVector<const TItemExprType*> items;
28472847
AddColumns(needRebuildSubLinks ? subLinkInputs : inputs, nullptr, refs, &qualifiedRefs, items, ctx.Expr);
2848+
if (items.empty()) {
2849+
newSorts.push_back(data.ChildPtr(index));
2850+
continue;
2851+
}
2852+
28482853
auto effectiveType = ctx.Expr.MakeType<TStructExprType>(items);
28492854
if (!effectiveType->Validate(oneSort->Pos(), ctx.Expr)) {
28502855
return false;
@@ -2871,7 +2876,7 @@ bool ValidateSort(TInputs& inputs, TInputs& subLinkInputs, const THashSet<TStrin
28712876
auto argNode = ctx.Expr.NewArgument(oneSort->Pos(), "row");
28722877
auto arguments = ctx.Expr.NewArguments(oneSort->Pos(), { argNode });
28732878
TExprNode::TPtr newRoot;
2874-
auto status = RebuildLambdaColumns(newLambda->TailPtr(), argNode, newRoot, inputs, nullptr, ctx);
2879+
auto status = RebuildLambdaColumns(newLambda->TailPtr(), argNode, newRoot, inputs, nullptr, ctx, usedInUsing);
28752880
if (status == IGraphTransformer::TStatus::Error) {
28762881
return false;
28772882
}
@@ -3766,7 +3771,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
37663771
auto argNode = ctx.Expr.NewArgument(data.Pos(), "row");
37673772
auto arguments = ctx.Expr.NewArguments(data.Pos(), { argNode });
37683773
TExprNode::TPtr newRoot;
3769-
auto status = RebuildLambdaColumns(newLambda->TailPtr(), argNode, newRoot, joinInputs, nullptr, ctx);
3774+
auto status = RebuildLambdaColumns(newLambda->TailPtr(), argNode, newRoot, joinInputs, nullptr, ctx, repeatedColumnsInUsing);
37703775
if (status == IGraphTransformer::TStatus::Error) {
37713776
return IGraphTransformer::TStatus::Error;
37723777
}
@@ -3862,9 +3867,12 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
38623867
bool needRewrite = false;
38633868
bool needRewriteUsing = false;
38643869
ui32 inputIndex = 0;
3870+
THashSet<TString> usedInUsingBefore;
38653871
for (ui32 joinGroupNo = 0; joinGroupNo < data.ChildrenSize(); ++joinGroupNo) {
38663872
joinInputs.push_back(inputs[inputIndex]);
38673873
++inputIndex;
3874+
// same names allowed in group, but not allowed in different since columns must not repeat
3875+
THashSet<TString> usedInUsingInThatGroup;
38683876
for (ui32 i = 0; i < data.Child(joinGroupNo)->ChildrenSize(); ++i) {
38693877
auto child = data.Child(joinGroupNo)->Child(i);
38703878
if (!EnsureTupleMinSize(*child, 1, ctx.Expr)) {
@@ -3918,10 +3926,11 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
39183926
needRewriteUsing = child->ChildrenSize() == 3;
39193927
for (ui32 i = 0; i < columnNames->ChildrenSize(); ++i) {
39203928
auto lcase = to_lower(TString(columnNames->Child(i)->Content()));
3921-
if (repeatedColumnsInUsing.contains(lcase)) {
3929+
if (usedInUsingBefore.contains(lcase)) {
39223930
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Duplicated column in USING clause: " << columnNames->Child(i)->Content()));
39233931
return IGraphTransformer::TStatus::Error;
39243932
}
3933+
usedInUsingInThatGroup.emplace(lcase);
39253934
repeatedColumnsInUsing.emplace(lcase, columnNames->Child(i)->Content());
39263935
}
39273936
} else {
@@ -3935,6 +3944,9 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
39353944
}
39363945
}
39373946
}
3947+
for (const auto& e: usedInUsingInThatGroup) {
3948+
usedInUsingBefore.emplace(e);
3949+
}
39383950
}
39393951

39403952
if (needRewrite || needRewriteUsing) {
@@ -4011,14 +4023,18 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
40114023
TExprNode::TListType lrNames(2);
40124024
auto pos = groupInputs[0].Type->FindItemI(name);
40134025
if (!pos) {
4026+
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Can't find column: " << name));
40144027
return IGraphTransformer::TStatus::Error;
40154028
}
4016-
lrNames[0] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[0].Alias, groupInputs[0].Type->GetItems()[*pos]->GetName()));
4029+
lrNames[0] = groupInputs.size() > 2
4030+
? ctx.Expr.NewList(inp->Pos(), {ctx.Expr.NewAtom(inp->Pos(), repeatedColumnsInUsing[to_lower(TString(name))])})
4031+
: ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[0].Alias, groupInputs[0].Type->GetItems()[*pos]->GetName()));
40174032
pos = groupInputs[1].Type->FindItemI(name);
40184033
if (!pos) {
4034+
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Can't find column: " << name));
40194035
return IGraphTransformer::TStatus::Error;
40204036
}
4021-
lrNames[1] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs[1].Alias, groupInputs[1].Type->GetItems()[*pos]->GetName()));
4037+
lrNames[1] = ctx.Expr.NewAtom(inp->Pos(), MakeAliasedColumn(groupInputs.back().Alias, groupInputs[1].Type->GetItems()[*pos]->GetName()));
40224038
nodes[colIdx] = ctx.Expr.NewList(inp->Pos(), std::move(lrNames));
40234039
}
40244040
TExprNode::TListType newJoin(4);
@@ -4073,7 +4089,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
40734089

40744090
TExprNode::TListType newGroups;
40754091
bool hasNewGroups = false;
4076-
if (!ValidateGroups(joinInputs, possibleAliases, data, ctx, newGroups, hasNewGroups, scanColumnsOnly, false, nullptr, "GROUP BY", &projectionOrders, result)) {
4092+
if (!ValidateGroups(joinInputs, possibleAliases, data, ctx, newGroups, hasNewGroups, scanColumnsOnly, false, nullptr, "GROUP BY", &projectionOrders, result, repeatedColumnsInUsing)) {
40774093
return IGraphTransformer::TStatus::Error;
40784094
}
40794095

@@ -4194,15 +4210,15 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
41944210
auto newChildren = x->ChildrenList();
41954211
TExprNode::TListType newGroups;
41964212
bool hasNewGroups = false;
4197-
if (!ValidateGroups(joinInputs, possibleAliases, *partitions, ctx, newGroups, hasNewGroups, scanColumnsOnly, true, groupExprs, "", nullptr, nullptr)) {
4213+
if (!ValidateGroups(joinInputs, possibleAliases, *partitions, ctx, newGroups, hasNewGroups, scanColumnsOnly, true, groupExprs, "", nullptr, nullptr, repeatedColumnsInUsing)) {
41984214
return IGraphTransformer::TStatus::Error;
41994215
}
42004216

42014217
newChildren[2] = ctx.Expr.NewList(x->Pos(), std::move(newGroups));
42024218

42034219
bool hasNewSort = false;
42044220
TExprNode::TListType newSorts;
4205-
if (!ValidateSort(joinInputs, joinInputs, possibleAliases, *sort, ctx, hasNewSort, newSorts, scanColumnsOnly, groupExprs, "", nullptr, nullptr)) {
4221+
if (!ValidateSort(joinInputs, joinInputs, possibleAliases, *sort, ctx, hasNewSort, newSorts, scanColumnsOnly, groupExprs, "", nullptr, nullptr, repeatedColumnsInUsing)) {
42064222
return IGraphTransformer::TStatus::Error;
42074223
}
42084224

@@ -4257,7 +4273,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
42574273
TExprNode::TListType newGroups;
42584274
TInputs projectionInputs;
42594275
projectionInputs.push_back(TInput{ "", outputRowType, Nothing(), TInput::Projection, {} });
4260-
if (!ValidateGroups(projectionInputs, {}, data, ctx, newGroups, hasNewGroups, scanColumnsOnly, false, nullptr, "DISTINCT ON", &projectionOrders, nullptr)) {
4276+
if (!ValidateGroups(projectionInputs, {}, data, ctx, newGroups, hasNewGroups, scanColumnsOnly, false, nullptr, "DISTINCT ON", &projectionOrders, nullptr, repeatedColumnsInUsing)) {
42614277
return IGraphTransformer::TStatus::Error;
42624278
}
42634279

@@ -4297,7 +4313,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
42974313
TExprNode::TListType newSortTupleItems;
42984314
// no effective types yet, scan lambda bodies
42994315
if (!ValidateSort(projectionInputs, joinInputs, possibleAliases, data, ctx, hasNewSort, newSortTupleItems,
4300-
scanColumnsOnly, groupExprs, "ORDER BY", &projectionOrders, GetSetting(options, "result"))) {
4316+
scanColumnsOnly, groupExprs, "ORDER BY", &projectionOrders, GetSetting(options, "result"), repeatedColumnsInUsing)) {
43014317
return IGraphTransformer::TStatus::Error;
43024318
}
43034319

‎ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2094,9 +2094,9 @@
20942094
],
20952095
"test.test[pg-join_using6-default.txt-Debug]": [
20962096
{
2097-
"checksum": "00bf2551654b7deff2a7132e0e9dde34",
2098-
"size": 700,
2099-
"uri": "https://{canondata_backend}/1937027/41c667e3bc192627f31ff3c007c7b7ae2889d42d/resource.tar.gz#test.test_pg-join_using6-default.txt-Debug_/opt.yql_patched"
2097+
"checksum": "65ae441a106b6d5404557323f59f52b5",
2098+
"size": 773,
2099+
"uri": "https://{canondata_backend}/1600758/3e1c972b67010ef976f8710e1ee2f4efbc022be3/resource.tar.gz#test.test_pg-join_using6-default.txt-Debug_/opt.yql_patched"
21002100
}
21012101
],
21022102
"test.test[pg-join_using6-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1831,9 +1831,9 @@
18311831
],
18321832
"test.test[pg-join_using3-default.txt-Debug]": [
18331833
{
1834-
"checksum": "c41e16066ab6a5b9340fb25232ea6f3b",
1835-
"size": 1314,
1836-
"uri": "https://{canondata_backend}/1936947/50ea69fb0cf990fe2d5d7c10fa7a0bbbe9836502/resource.tar.gz#test.test_pg-join_using3-default.txt-Debug_/opt.yql_patched"
1834+
"checksum": "6c13d66a25990035579704ec34db4a72",
1835+
"size": 1384,
1836+
"uri": "https://{canondata_backend}/1925842/1b41163832bf0b025ac6bbe8bddee67fd3dcc69d/resource.tar.gz#test.test_pg-join_using3-default.txt-Debug_/opt.yql_patched"
18371837
}
18381838
],
18391839
"test.test[pg-join_using3-default.txt-Plan]": [

0 commit comments

Comments
 (0)