Skip to content

Commit ac47ebc

Browse files
avevadavevad
authored
Merge 7808bd9 into 4c31460 
2 parents 4c31460 + 7808bd9 commit ac47ebc

File tree

7 files changed

+228
-0
lines changed

7 files changed

+228
-0
lines changed

‎ydb/library/yql/core/type_ann/type_ann_core.cpp

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6074,6 +6074,141 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
60746074
return IGraphTransformer::TStatus::Ok;
60756075
}
60766076

6077+
IGraphTransformer::TStatus StructMergeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
6078+
if (!EnsureMinMaxArgsCount(*input, 2, 3, ctx.Expr)) {
6079+
return IGraphTransformer::TStatus::Error;
6080+
}
6081+
6082+
auto left = input->Child(0);
6083+
auto right = input->Child(1);
6084+
6085+
if (HasError(left->GetTypeAnn(), ctx.Expr)) {
6086+
return IGraphTransformer::TStatus::Error;
6087+
}
6088+
6089+
if (HasError(right->GetTypeAnn(), ctx.Expr)) {
6090+
return IGraphTransformer::TStatus::Error;
6091+
}
6092+
6093+
if (!EnsureStructType(*left, ctx.Expr)) {
6094+
return IGraphTransformer::TStatus::Error;
6095+
}
6096+
auto leftType = left->GetTypeAnn()->Cast<TStructExprType>();
6097+
6098+
if (!EnsureStructType(*right, ctx.Expr)) {
6099+
return IGraphTransformer::TStatus::Error;
6100+
}
6101+
auto rightType = right->GetTypeAnn()->Cast<TStructExprType>();
6102+
6103+
TExprNode::TPtr mergeLambda = nullptr;
6104+
if (input->ChildrenSize() == 3) {
6105+
mergeLambda = input->ChildPtr(2);
6106+
auto status = ConvertToLambda(mergeLambda, ctx.Expr, 3);
6107+
if (status.Level != IGraphTransformer::TStatus::Ok) {
6108+
return status;
6109+
}
6110+
} else {
6111+
mergeLambda = ctx.Expr.Builder(input->Pos())
6112+
.Lambda()
6113+
.Param("name")
6114+
.Param("left")
6115+
.Param("right")
6116+
.Callable("Unwrap")
6117+
.Callable(0, "Coalesce")
6118+
.Arg(0, "left")
6119+
.Arg(1, "right")
6120+
.Seal()
6121+
.Seal()
6122+
.Seal()
6123+
.Build();
6124+
}
6125+
6126+
auto buildJustMember = [&ctx, &input](const TExprNode::TPtr &st, const TStringBuf& name) -> TExprNode::TPtr {
6127+
return ctx.Expr.Builder(input->Pos())
6128+
.Callable("Just")
6129+
.Callable(0, "Member")
6130+
.Add(0, st)
6131+
.Atom(1, name)
6132+
.Seal()
6133+
.Seal()
6134+
.Build();
6135+
};
6136+
6137+
auto mergeMembers = [&ctx, &buildJustMember, &input, &left, &right, &mergeLambda](const TStringBuf& name, bool hasLeft, bool hasRight) -> TExprNode::TPtr {
6138+
auto leftMaybe = hasLeft ?
6139+
buildJustMember(left, name) :
6140+
ctx.Expr.NewCallable(input->Pos(), "Nothing", {
6141+
ExpandType(input->Pos(), *ctx.Expr.MakeType<TOptionalExprType>(right->GetTypeAnn()->Cast<TStructExprType>()->FindItemType(name)), ctx.Expr)
6142+
});
6143+
6144+
auto rightMaybe = hasRight ?
6145+
buildJustMember(right, name) :
6146+
ctx.Expr.NewCallable(input->Pos(), "Nothing", {
6147+
ExpandType(input->Pos(), *ctx.Expr.MakeType<TOptionalExprType>(left->GetTypeAnn()->Cast<TStructExprType>()->FindItemType(name)), ctx.Expr)
6148+
});
6149+
6150+
return ctx.Expr.Builder(input->Pos())
6151+
.List()
6152+
.Atom(0, name)
6153+
.Apply(1, mergeLambda)
6154+
.With(0)
6155+
.Callable("String")
6156+
.Atom(0, name)
6157+
.Seal()
6158+
.Done()
6159+
.With(1, leftMaybe)
6160+
.With(2, rightMaybe)
6161+
.Seal()
6162+
.Seal()
6163+
.Build();
6164+
};
6165+
6166+
TExprNode::TListType children;
6167+
6168+
bool isUnion = input->Content() == "StructUnion";
6169+
bool isIntersection = input->Content() == "StructIntersection";
6170+
bool isDifference = input->Content() == "StructDifference";
6171+
bool isSymmDifference = input->Content() == "StructSymmetricDifference";
6172+
6173+
for (const auto* leftItem : leftType->GetItems()) {
6174+
const auto& name = leftItem->GetName();
6175+
if (isUnion) {
6176+
if (rightType->FindItem(name)) {
6177+
children.push_back(mergeMembers(name, true, true));
6178+
} else {
6179+
children.push_back(mergeMembers(name, true, false));
6180+
}
6181+
}
6182+
if (isIntersection) {
6183+
if (rightType->FindItem(name)) {
6184+
children.push_back(mergeMembers(name, true, true));
6185+
}
6186+
}
6187+
if (isDifference || isSymmDifference) {
6188+
if (!rightType->FindItem(name)) {
6189+
children.push_back(mergeMembers(name, true, false));
6190+
}
6191+
}
6192+
}
6193+
6194+
for (const auto* rightItem : rightType->GetItems()) {
6195+
const auto& name = rightItem->GetName();
6196+
if (isUnion) {
6197+
if (!leftType->FindItem(name)) {
6198+
children.push_back(mergeMembers(name, false, true));
6199+
}
6200+
}
6201+
if (isSymmDifference) {
6202+
if (!leftType->FindItem(name)) {
6203+
children.push_back(mergeMembers(name, false, true));
6204+
}
6205+
}
6206+
}
6207+
6208+
output = ctx.Expr.NewCallable(input->Pos(), "AsStruct", std::move(children));
6209+
return IGraphTransformer::TStatus::Repeat;
6210+
}
6211+
60776212
IGraphTransformer::TStatus StaticMapWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
60786213
if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
60796214
return IGraphTransformer::TStatus::Error;
@@ -12238,6 +12373,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
1223812373
Functions["PgGrouping"] = &PgGroupingWrapper;
1223912374
Functions["PgGroupingSet"] = &PgGroupingSetWrapper;
1224012375
Functions["PgToRecord"] = &PgToRecordWrapper;
12376+
Functions["StructUnion"] = &StructMergeWrapper;
12377+
Functions["StructIntersection"] = &StructMergeWrapper;
12378+
Functions["StructDifference"] = &StructMergeWrapper;
12379+
Functions["StructSymmetricDifference"] = &StructMergeWrapper;
1224112380

1224212381
Functions["AutoDemux"] = &AutoDemuxWrapper;
1224312382
Functions["AggrCountInit"] = &AggrCountInitWrapper;

‎ydb/library/yql/sql/v1/builtin.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3070,6 +3070,10 @@ struct TBuiltinFuncData {
30703070
{"flattenmembers", BuildNamedBuiltinFactoryCallback<TFlattenMembers>("FlattenMembers")},
30713071
{"staticmap", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticMap", 2, 2) },
30723072
{"staticzip", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticZip", 1, -1) },
3073+
{"structunion", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StructUnion", 2, 3)},
3074+
{"structintersection", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StructIntersection", 2, 3)},
3075+
{"structdifference", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StructDifference", 2, 3)},
3076+
{"structsymmetricdifference", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StructSymmetricDifference", 2, 3)},
30733077
{"staticfold", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticFold", 3, 3)},
30743078
{"staticfold1", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("StaticFold1", 3, 3)},
30753079

‎ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -975,6 +975,28 @@
975975
"uri": "file://test.test_expr-non_persistable_group_by_column_fail--Results_/extracted"
976976
}
977977
],
978+
"test.test[expr-struct_merge-default.txt-Analyze]": [
979+
{
980+
"checksum": "b4dd508a329723c74293d80f0278c705",
981+
"size": 505,
982+
"uri": "https://{canondata_backend}/1775059/79f40817d9be6347f8a0a937bdd3c46c326ab7d3/resource.tar.gz#test.test_expr-struct_merge-default.txt-Analyze_/plan.txt"
983+
}
984+
],
985+
"test.test[expr-struct_merge-default.txt-Debug]": [
986+
{
987+
"checksum": "7ed8bb90b0fd6a7a9c734d5e24ec3a79",
988+
"size": 867,
989+
"uri": "https://{canondata_backend}/1689644/d939c79f1c25569f7b8f4e5b740e070ad72d7ad7/resource.tar.gz#test.test_expr-struct_merge-default.txt-Debug_/opt.yql_patched"
990+
}
991+
],
992+
"test.test[expr-struct_merge-default.txt-Plan]": [
993+
{
994+
"checksum": "b4dd508a329723c74293d80f0278c705",
995+
"size": 505,
996+
"uri": "https://{canondata_backend}/1775059/79f40817d9be6347f8a0a937bdd3c46c326ab7d3/resource.tar.gz#test.test_expr-struct_merge-default.txt-Plan_/plan.txt"
997+
}
998+
],
999+
"test.test[expr-struct_merge-default.txt-Results]": [],
9781000
"test.test[flatten_by-flatten_one_field--Analyze]": [
9791001
{
9801002
"checksum": "dfeb435c40cd1a0a98c74310e1507366",

‎ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,20 @@
923923
"uri": "https://{canondata_backend}/1775059/3cb7d014d70b84dbcb84645fa987dd9d47d7fd6c/resource.tar.gz#test.test_expr-many_opt_comp-default.txt-Plan_/plan.txt"
924924
}
925925
],
926+
"test.test[expr-struct_merge-default.txt-Debug]": [
927+
{
928+
"checksum": "ab77466270296597939428807f0af395",
929+
"size": 866,
930+
"uri": "https://{canondata_backend}/1942415/ecf45b8d311b13ba55e2de94295cabed9b642863/resource.tar.gz#test.test_expr-struct_merge-default.txt-Debug_/opt.yql_patched"
931+
}
932+
],
933+
"test.test[expr-struct_merge-default.txt-Plan]": [
934+
{
935+
"checksum": "b4dd508a329723c74293d80f0278c705",
936+
"size": 505,
937+
"uri": "https://{canondata_backend}/1936842/e15468da5c6a430935df259a2106604daa68ad66/resource.tar.gz#test.test_expr-struct_merge-default.txt-Plan_/plan.txt"
938+
}
939+
],
926940
"test.test[expr-uuid_order-default.txt-Debug]": [
927941
{
928942
"checksum": "dd888f0b22d793979dbf237917d203dd",

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5977,6 +5977,13 @@
59775977
"uri": "https://{canondata_backend}/1784117/d56ae82ad9d30397a41490647be1bd2124718f98/resource.tar.gz#test_sql2yql.test_expr-struct_literal_members_/sql.yql"
59785978
}
59795979
],
5980+
"test_sql2yql.test[expr-struct_merge]": [
5981+
{
5982+
"checksum": "e3781bd00212a17b07691294caa0c1b0",
5983+
"size": 3095,
5984+
"uri": "https://{canondata_backend}/1916746/116cafe28e270e7917dbeab5e0d1b5f2357e5c16/resource.tar.gz#test_sql2yql.test_expr-struct_merge_/sql.yql"
5985+
}
5986+
],
59805987
"test_sql2yql.test[expr-struct_slice]": [
59815988
{
59825989
"checksum": "4d0f79865e785d3f3b0e9e0110bb1f86",
@@ -24401,6 +24408,13 @@
2440124408
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_expr-struct_literal_members_/formatted.sql"
2440224409
}
2440324410
],
24411+
"test_sql_format.test[expr-struct_merge]": [
24412+
{
24413+
"checksum": "509cfc4518e9c467b2cd05a5e568c00b",
24414+
"size": 413,
24415+
"uri": "https://{canondata_backend}/1916746/116cafe28e270e7917dbeab5e0d1b5f2357e5c16/resource.tar.gz#test_sql_format.test_expr-struct_merge_/formatted.sql"
24416+
}
24417+
],
2440424418
"test_sql_format.test[expr-struct_slice]": [
2440524419
{
2440624420
"checksum": "8a9f027371f1722b5753e5b53cf62346",

‎ydb/library/yql/tests/sql/suites/expr/struct_merge.sql

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/* syntax version 1 */
2+
3+
$merge = ($_name, $l, $r) -> { return Coalesce($l, 0) + Coalesce($r, 0); };
4+
$left = <|a: 1, b: 2, c: 3|>;
5+
$right = <|c: 1, d: 2, e: 3|>;
6+
7+
SELECT
8+
StructUnion($left, $right),
9+
StructUnion($left, $right, $merge),
10+
StructIntersection($left, $right),
11+
StructIntersection($left, $right, $merge),
12+
StructDifference($left, $right),
13+
StructSymmetricDifference($left, $right)
14+
;

‎ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,6 +1003,27 @@
10031003
"uri": "file://test.test_expr-non_persistable_group_by_column_fail--Results_/extracted"
10041004
}
10051005
],
1006+
"test.test[expr-struct_merge-default.txt-Debug]": [
1007+
{
1008+
"checksum": "54a15a62a123d1a72d9190f26324aa13",
1009+
"size": 796,
1010+
"uri": "https://{canondata_backend}/1784826/6c4ac0f02ea872d52d4b59ee9d0f2b2963fe6800/resource.tar.gz#test.test_expr-struct_merge-default.txt-Debug_/opt.yql"
1011+
}
1012+
],
1013+
"test.test[expr-struct_merge-default.txt-Plan]": [
1014+
{
1015+
"checksum": "b4dd508a329723c74293d80f0278c705",
1016+
"size": 505,
1017+
"uri": "https://{canondata_backend}/1881367/84017fd57088f9d554efcf1a44f82b22e5b164b7/resource.tar.gz#test.test_expr-struct_merge-default.txt-Plan_/plan.txt"
1018+
}
1019+
],
1020+
"test.test[expr-struct_merge-default.txt-Results]": [
1021+
{
1022+
"checksum": "83bf749394c8035e5f04d3cf2e23c44c",
1023+
"size": 9246,
1024+
"uri": "https://{canondata_backend}/1784826/6c4ac0f02ea872d52d4b59ee9d0f2b2963fe6800/resource.tar.gz#test.test_expr-struct_merge-default.txt-Results_/results.txt"
1025+
}
1026+
],
10061027
"test.test[flatten_by-flatten_one_field--Debug]": [
10071028
{
10081029
"checksum": "1e1f4bdf8614f3314eb9a5b53d01d8db",

0 commit comments

Comments
 (0)