Skip to content

Commit ccd2ae0

Browse files
vitstnvitstn
authored
lineage: support of isolated subgraph (#3848)
1 parent ff494f6 commit ccd2ae0

File tree

6 files changed

+93
-0
lines changed

6 files changed

+93
-0
lines changed

‎ydb/library/yql/core/services/yql_lineage.cpp‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ class TLineageScanner {
2323
for (auto& p : Ctx_.DataSources) {
2424
if (p->IsRead(node)) {
2525
Reads_[&node] = p.Get();
26+
HasReads_.emplace(&node);
2627
}
2728
}
2829

@@ -32,6 +33,15 @@ class TLineageScanner {
3233
}
3334
}
3435

36+
return true;
37+
}, [&](const TExprNode& node) {
38+
for (const auto& child : node.Children()) {
39+
if (HasReads_.contains(child.Get())) {
40+
HasReads_.emplace(&node);
41+
break;
42+
}
43+
}
44+
3545
return true;
3646
});
3747

@@ -210,6 +220,27 @@ class TLineageScanner {
210220
return &lineage;
211221
}
212222

223+
if (!HasReads_.contains(&node)) {
224+
auto type = node.GetTypeAnn();
225+
if (type->GetKind() == ETypeAnnotationKind::List) {
226+
auto itemType = type->Cast<TListExprType>()->GetItemType();
227+
if (itemType->GetKind() == ETypeAnnotationKind::Struct) {
228+
auto structType = itemType->Cast<TStructExprType>();
229+
lineage.Fields.ConstructInPlace();
230+
for (const auto& i : structType->GetItems()) {
231+
if (i->GetName().StartsWith("_yql_sys_")) {
232+
continue;
233+
}
234+
235+
TString fieldName(i->GetName());
236+
(*lineage.Fields).emplace(fieldName, TFieldsLineage());
237+
}
238+
239+
return &lineage;
240+
}
241+
}
242+
}
243+
213244
if (node.IsCallable({
214245
"Unordered",
215246
"UnorderedSubquery",
@@ -828,6 +859,7 @@ class TLineageScanner {
828859
TNodeMap<TVector<ui32>> ReadIds_;
829860
TNodeMap<ui32> WriteIds_;
830861
TNodeMap<TLineage> Lineages_;
862+
TNodeSet HasReads_;
831863
};
832864

833865
}

‎ydb/library/yql/core/yql_expr_optimize.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -898,6 +898,11 @@ void VisitExpr(const TExprNode& root, const TExprVisitRefFunc& func) {
898898
VisitExprInternal(root, func, {}, visitedNodes);
899899
}
900900

901+
void VisitExpr(const TExprNode& root, const TExprVisitRefFunc& preFunc, const TExprVisitRefFunc& postFunc) {
902+
TNodeSet visitedNodes;
903+
VisitExprInternal(root, preFunc, postFunc, visitedNodes);
904+
}
905+
901906
void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes) {
902907
VisitExprInternal(root, func, {}, visitedNodes);
903908
}

‎ydb/library/yql/core/yql_expr_optimize.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func);
5858
void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preFunc, const TExprVisitPtrFunc& postFunc);
5959
void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes);
6060
void VisitExpr(const TExprNode& root, const TExprVisitRefFunc& func);
61+
void VisitExpr(const TExprNode& root, const TExprVisitRefFunc& preFunc, const TExprVisitRefFunc& postFunc);
6162
void VisitExprLambdasLast(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preLambdaFunc, const TExprVisitPtrFunc& postLambdaFunc);
6263

6364

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9862,6 +9862,13 @@
98629862
"uri": "https://{canondata_backend}/1924537/1ab444909086b08bd4fe21c5a43f5e183c647e0a/resource.tar.gz#test_sql2yql.test_lineage-if_struct_/sql.yql"
98639863
}
98649864
],
9865+
"test_sql2yql.test[lineage-isolated]": [
9866+
{
9867+
"checksum": "bf02803998f7f9a0ff1e4226e769b8a6",
9868+
"size": 1857,
9869+
"uri": "https://{canondata_backend}/1889210/c8b4fae053b51029d1889b70ffd7fe56013cabc7/resource.tar.gz#test_sql2yql.test_lineage-isolated_/sql.yql"
9870+
}
9871+
],
98659872
"test_sql2yql.test[lineage-join_as_struct]": [
98669873
{
98679874
"checksum": "9fda95e8ba4e26de0dc24caa99de6d16",
@@ -28181,6 +28188,13 @@
2818128188
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_lineage-if_struct_/formatted.sql"
2818228189
}
2818328190
],
28191+
"test_sql_format.test[lineage-isolated]": [
28192+
{
28193+
"checksum": "d239bb9b177cc40755c6f66be76a5b98",
28194+
"size": 160,
28195+
"uri": "https://{canondata_backend}/1889210/c8b4fae053b51029d1889b70ffd7fe56013cabc7/resource.tar.gz#test_sql_format.test_lineage-isolated_/formatted.sql"
28196+
}
28197+
],
2818428198
"test_sql_format.test[lineage-join_as_struct]": [
2818528199
{
2818628200
"checksum": "a5223ab2032510ed8523fb48abf94904",

‎ydb/library/yql/tests/sql/suites/lineage/isolated.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
USE plato;
2+
3+
$one = select dummy from as_table([<|dummy:1|>]);
4+
5+
INSERT INTO Output
6+
SELECT key FROM Input AS a
7+
CROSS JOIN $one AS b;
8+

‎ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1829,6 +1829,39 @@
18291829
"uri": "https://{canondata_backend}/1775059/105879cd875204bc7c66426c8bfdd2bbd01bfaf7/resource.tar.gz#test.test_like-regexp_clause--Results_/results.txt"
18301830
}
18311831
],
1832+
"test.test[lineage-isolated-default.txt-Debug]": [
1833+
{
1834+
"checksum": "8028b91e2314dab0c488f0757abbdb47",
1835+
"size": 3324,
1836+
"uri": "https://{canondata_backend}/1925842/653a2f99a577c31a8b1f5618a0939a6f807664e8/resource.tar.gz#test.test_lineage-isolated-default.txt-Debug_/opt.yql"
1837+
}
1838+
],
1839+
"test.test[lineage-isolated-default.txt-Lineage]": [
1840+
{
1841+
"checksum": "76522ff9c85d6989407abd6a1aff0859",
1842+
"size": 922,
1843+
"uri": "https://{canondata_backend}/1925842/653a2f99a577c31a8b1f5618a0939a6f807664e8/resource.tar.gz#test.test_lineage-isolated-default.txt-Lineage_/results.txt"
1844+
}
1845+
],
1846+
"test.test[lineage-isolated-default.txt-Plan]": [
1847+
{
1848+
"checksum": "b4bcb9030b738191f1709be4ad187678",
1849+
"size": 6371,
1850+
"uri": "https://{canondata_backend}/1925842/653a2f99a577c31a8b1f5618a0939a6f807664e8/resource.tar.gz#test.test_lineage-isolated-default.txt-Plan_/plan.txt"
1851+
}
1852+
],
1853+
"test.test[lineage-isolated-default.txt-Results]": [
1854+
{
1855+
"checksum": "ed8c973d4e78fa58812b66f36d592b55",
1856+
"size": 92,
1857+
"uri": "https://{canondata_backend}/1925842/653a2f99a577c31a8b1f5618a0939a6f807664e8/resource.tar.gz#test.test_lineage-isolated-default.txt-Results_/Output.txt"
1858+
},
1859+
{
1860+
"checksum": "b4954e4a3eb891d8d79720abeee5cacf",
1861+
"size": 373,
1862+
"uri": "https://{canondata_backend}/1925842/653a2f99a577c31a8b1f5618a0939a6f807664e8/resource.tar.gz#test.test_lineage-isolated-default.txt-Results_/Output.yqlrun.txt.attr"
1863+
}
1864+
],
18321865
"test.test[optimizers-passthrough_sortness_over_map-default.txt-Debug]": [
18331866
{
18341867
"checksum": "e0939502d3c57798c9332b5390cc063a",

0 commit comments

Comments
 (0)