Skip to content

Commit 4b9626d

Browse files
pashandor789pashandor789
authored
Merge a2d3033 into ded1634 
2 parents ded1634 + a2d3033 commit 4b9626d

File tree

502 files changed

+2648
-4030
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

502 files changed

+2648
-4030
lines changed

‎ydb/core/kqp/opt/kqp_opt_build_txs.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,7 @@ class TKqpBuildTxsTransformer : public TSyncTransformerBase {
576576
.Add(CreateKqpBuildPhyStagesTransformer(/* allowDependantConsumers */ false, typesCtx, config->BlockChannelsMode), "BuildPhysicalStages")
577577
.Add(CreateKqpBuildWideBlockChannelsTransformer(typesCtx, config->BlockChannelsMode), "BuildWideBlockChannels")
578578
.Add(*BuildTxTransformer, "BuildPhysicalTx")
579-
.Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false), "Peephole")
579+
.Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false, {"KqpPeephole-RewriteCrossJoin"}), "Peephole")
580580
.Build(false);
581581

582582
ScanTxTransformer = TTransformationPipeline(&typesCtx)
@@ -586,7 +586,7 @@ class TKqpBuildTxsTransformer : public TSyncTransformerBase {
586586
.AddPostTypeAnnotation(/* forSubgraph */ true)
587587
.Add(CreateKqpBuildPhyStagesTransformer(config->SpillingEnabled(), typesCtx, config->BlockChannelsMode), "BuildPhysicalStages")
588588
.Add(*BuildTxTransformer, "BuildPhysicalTx")
589-
.Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false), "Peephole")
589+
.Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false, {"KqpPeephole-RewriteCrossJoin"}), "Peephole")
590590
.Build(false);
591591
}
592592

‎ydb/core/kqp/opt/kqp_query_plan.cpp‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,6 +1058,8 @@ class TxPlanSerializer {
10581058
operatorId = Visit(maybeDelete.Cast(), planNode);
10591059
} else if (auto maybeArg = TMaybeNode<TCoArgument>(node)) {
10601060
return {CurrentArgContext.AddArg(node.Get())};
1061+
} else if (auto maybeCrossJoin = TMaybeNode<TDqPhyCrossJoin>(node)) {
1062+
operatorId = Visit(maybeCrossJoin.Cast(), planNode);
10611063
}
10621064

10631065
TVector<std::variant<ui32, TArgContext>> inputIds;
@@ -1374,6 +1376,15 @@ class TxPlanSerializer {
13741376
return operatorId;
13751377
}
13761378

1379+
std::variant<ui32, TArgContext> Visit(const TDqPhyCrossJoin&, TQueryPlanNode& planNode) {
1380+
const auto name = "CrossJoin";
1381+
1382+
TOperator op;
1383+
op.Properties["Name"] = name;
1384+
1385+
return AddOperator(planNode, name, std::move(op));
1386+
}
1387+
13771388
std::variant<ui32, TArgContext> Visit(const TCoMapJoinCore& join, TQueryPlanNode& planNode) {
13781389
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)";
13791390

@@ -2055,11 +2066,6 @@ NJson::TJsonValue ReconstructQueryPlanRec(const NJson::TJsonValue& plan,
20552066
auto inputPlanId = opInput.GetMapSafe().at("InternalOperatorId").GetIntegerSafe();
20562067
planInputs.push_back( ReconstructQueryPlanRec(plan, inputPlanId, planIndex, precomputes, nodeCounter));
20572068
}
2058-
2059-
// Sometimes we have multiple inputs for these operators, break after the first one
2060-
if (opName == "Filter" || opName == "TopSort" || opName == "Aggregate") {
2061-
break;
2062-
}
20632069
}
20642070

20652071
if (op.GetMapSafe().contains("Inputs")) {
@@ -2085,12 +2091,9 @@ NJson::TJsonValue ReconstructQueryPlanRec(const NJson::TJsonValue& plan,
20852091
maybePrecompute = op.GetMapSafe().at("Iterator").GetStringSafe();
20862092
}
20872093

2088-
if (precomputes.contains(maybePrecompute)) {
2089-
//YQL_CLOG(TRACE, CoreDq) << "Following precompute: " << maybePrecompute ;
2094+
if (precomputes.contains(maybePrecompute) && planInputs.empty()) {
20902095
planInputs.push_back(ReconstructQueryPlanRec(precomputes.at(maybePrecompute), 0, planIndex, precomputes, nodeCounter));
2091-
} //else {
2092-
// YQL_CLOG(TRACE, CoreDq) << "Didn't find precompute: " << maybePrecompute ;
2093-
//}
2096+
}
20942097
}
20952098

20962099
result["Node Type"] = opName;

‎ydb/core/kqp/opt/peephole/kqp_opt_peephole.cpp‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ TStatus ReplaceNonDetFunctionsWithParams(TExprNode::TPtr& input, TExprContext& c
8686

8787
class TKqpPeepholeTransformer : public TOptimizeTransformerBase {
8888
public:
89-
TKqpPeepholeTransformer(TTypeAnnotationContext& typesCtx)
90-
: TOptimizeTransformerBase(&typesCtx, NYql::NLog::EComponent::ProviderKqp, {})
89+
TKqpPeepholeTransformer(TTypeAnnotationContext& typesCtx, TSet<TString> disabledOpts)
90+
: TOptimizeTransformerBase(&typesCtx, NYql::NLog::EComponent::ProviderKqp, disabledOpts)
9191
{
9292
#define HNDL(name) "KqpPeephole-"#name, Hndl(&TKqpPeepholeTransformer::name)
9393
AddHandler(0, &TDqReplicate::Match, HNDL(RewriteReplicate));
@@ -152,8 +152,12 @@ class TKqpPeepholeTransformer : public TOptimizeTransformerBase {
152152
};
153153

154154
struct TKqpPeepholePipelineConfigurator : IPipelineConfigurator {
155-
TKqpPeepholePipelineConfigurator(TKikimrConfiguration::TPtr config)
155+
TKqpPeepholePipelineConfigurator(
156+
TKikimrConfiguration::TPtr config,
157+
TSet<TString> disabledOpts
158+
)
156159
: Config(config)
160+
, DisabledOpts(disabledOpts)
157161
{}
158162

159163
void AfterCreate(TTransformationPipeline*) const override {
@@ -163,11 +167,12 @@ struct TKqpPeepholePipelineConfigurator : IPipelineConfigurator {
163167
}
164168

165169
void AfterOptimize(TTransformationPipeline* pipeline) const override {
166-
pipeline->Add(new TKqpPeepholeTransformer(*pipeline->GetTypeAnnotationContext()), "KqpPeephole");
170+
pipeline->Add(new TKqpPeepholeTransformer(*pipeline->GetTypeAnnotationContext(), DisabledOpts), "KqpPeephole");
167171
}
168172

169173
private:
170174
TKikimrConfiguration::TPtr Config;
175+
TSet<TString> DisabledOpts;
171176
};
172177

173178
class TKqpPeepholeFinalTransformer : public TOptimizeTransformerBase {
@@ -210,9 +215,9 @@ struct TKqpPeepholePipelineFinalConfigurator : IPipelineConfigurator {
210215

211216
TStatus PeepHoleOptimize(const TExprBase& program, TExprNode::TPtr& newProgram, TExprContext& ctx,
212217
IGraphTransformer& typeAnnTransformer, TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config,
213-
bool allowNonDeterministicFunctions, bool withFinalStageRules)
218+
bool allowNonDeterministicFunctions, bool withFinalStageRules, TSet<TString> disabledOpts)
214219
{
215-
TKqpPeepholePipelineConfigurator kqpPeephole(config);
220+
TKqpPeepholePipelineConfigurator kqpPeephole(config, disabledOpts);
216221
TKqpPeepholePipelineFinalConfigurator kqpPeepholeFinal(config);
217222
TPeepholeSettings peepholeSettings;
218223
peepholeSettings.CommonConfig = &kqpPeephole;
@@ -237,7 +242,7 @@ TStatus PeepHoleOptimize(const TExprBase& program, TExprNode::TPtr& newProgram,
237242

238243
TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprContext& ctx,
239244
IGraphTransformer& typeAnnTransformer, TTypeAnnotationContext& typesCtx, THashSet<ui64>& optimizedStages,
240-
TKikimrConfiguration::TPtr config, bool withFinalStageRules)
245+
TKikimrConfiguration::TPtr config, bool withFinalStageRules, TSet<TString> disabledOpts)
241246
{
242247
TVector<TDqPhyStage> stages;
243248
stages.reserve(tx.Stages().Size());
@@ -266,7 +271,7 @@ TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprConte
266271

267272
TExprNode::TPtr newProgram;
268273
auto status = PeepHoleOptimize(program, newProgram, ctx, typeAnnTransformer, typesCtx, config,
269-
allowNonDeterministicFunctions, withFinalStageRules);
274+
allowNonDeterministicFunctions, withFinalStageRules, disabledOpts);
270275
if (status != TStatus::Ok) {
271276
ctx.AddError(TIssue(ctx.GetPosition(stage.Pos()), "Peephole optimization failed for KQP transaction"));
272277
return {};
@@ -311,12 +316,18 @@ TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprConte
311316

312317
class TKqpTxPeepholeTransformer : public TSyncTransformerBase {
313318
public:
314-
TKqpTxPeepholeTransformer(IGraphTransformer* typeAnnTransformer,
315-
TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config, bool withFinalStageRules)
319+
TKqpTxPeepholeTransformer(
320+
IGraphTransformer* typeAnnTransformer,
321+
TTypeAnnotationContext& typesCtx,
322+
TKikimrConfiguration::TPtr config,
323+
bool withFinalStageRules,
324+
TSet<TString> disabledOpts
325+
)
316326
: TypeAnnTransformer(typeAnnTransformer)
317327
, TypesCtx(typesCtx)
318328
, Config(config)
319329
, WithFinalStageRules(withFinalStageRules)
330+
, DisabledOpts(disabledOpts)
320331
{}
321332

322333
TStatus DoTransform(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) final {
@@ -334,7 +345,7 @@ class TKqpTxPeepholeTransformer : public TSyncTransformerBase {
334345
auto tx = input.Cast<TKqpPhysicalTx>();
335346

336347
THashSet<ui64> optimizedStages;
337-
auto optimizedTx = PeepholeOptimize(tx, ctx, *TypeAnnTransformer, TypesCtx, optimizedStages, Config, WithFinalStageRules);
348+
auto optimizedTx = PeepholeOptimize(tx, ctx, *TypeAnnTransformer, TypesCtx, optimizedStages, Config, WithFinalStageRules, DisabledOpts);
338349

339350
if (!optimizedTx) {
340351
return TStatus::Error;
@@ -356,6 +367,7 @@ class TKqpTxPeepholeTransformer : public TSyncTransformerBase {
356367
TKikimrConfiguration::TPtr Config;
357368
bool Optimized = false;
358369
bool WithFinalStageRules = true;
370+
TSet<TString> DisabledOpts;
359371
};
360372

361373
class TKqpTxsPeepholeTransformer : public TSyncTransformerBase {
@@ -430,14 +442,22 @@ class TKqpTxsPeepholeTransformer : public TSyncTransformerBase {
430442

431443
} // anonymous namespace
432444

433-
TAutoPtr<IGraphTransformer> CreateKqpTxPeepholeTransformer(NYql::IGraphTransformer* typeAnnTransformer,
434-
TTypeAnnotationContext& typesCtx, const TKikimrConfiguration::TPtr& config, bool withFinalStageRules)
445+
TAutoPtr<IGraphTransformer> CreateKqpTxPeepholeTransformer(
446+
NYql::IGraphTransformer* typeAnnTransformer,
447+
TTypeAnnotationContext& typesCtx,
448+
const TKikimrConfiguration::TPtr& config,
449+
bool withFinalStageRules,
450+
TSet<TString> disabledOpts
451+
)
435452
{
436-
return new TKqpTxPeepholeTransformer(typeAnnTransformer, typesCtx, config, withFinalStageRules);
453+
return new TKqpTxPeepholeTransformer(typeAnnTransformer, typesCtx, config, withFinalStageRules, disabledOpts);
437454
}
438455

439-
TAutoPtr<IGraphTransformer> CreateKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer,
440-
TTypeAnnotationContext& typesCtx, const TKikimrConfiguration::TPtr& config)
456+
TAutoPtr<IGraphTransformer> CreateKqpTxsPeepholeTransformer(
457+
TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer,
458+
TTypeAnnotationContext& typesCtx,
459+
const TKikimrConfiguration::TPtr& config
460+
)
441461
{
442462
return new TKqpTxsPeepholeTransformer(std::move(typeAnnTransformer), typesCtx, config);
443463
}

‎ydb/core/kqp/opt/peephole/kqp_opt_peephole.h‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,18 @@
44

55
namespace NKikimr::NKqp::NOpt {
66

7-
TAutoPtr<NYql::IGraphTransformer> CreateKqpTxPeepholeTransformer(NYql::IGraphTransformer* typeAnnTransformer,
8-
NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config, bool withFinalStageRules = true);
7+
TAutoPtr<NYql::IGraphTransformer> CreateKqpTxPeepholeTransformer(
8+
NYql::IGraphTransformer* typeAnnTransformer,
9+
NYql::TTypeAnnotationContext& typesCtx,
10+
const NYql::TKikimrConfiguration::TPtr& config,
11+
bool withFinalStageRules = true,
12+
TSet<TString> disabledOpts = {}
13+
);
914

10-
TAutoPtr<NYql::IGraphTransformer> CreateKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer,
11-
NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config);
15+
TAutoPtr<NYql::IGraphTransformer> CreateKqpTxsPeepholeTransformer(
16+
TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer,
17+
NYql::TTypeAnnotationContext& typesCtx,
18+
const NYql::TKikimrConfiguration::TPtr& config
19+
);
1220

1321
} // namespace NKikimr::NKqp::NOpt

‎ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_2.plan‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,10 @@
3030
"Plans": [
3131
{
3232
"CTE Name": "precompute_0_0",
33-
"Node Type": "Limit-Aggregate-ConstantExpr-InnerJoin (MapJoin)-ConstantExpr-Filter-TableFullScan",
33+
"Node Type": "Limit-CrossJoin-InnerJoin (MapJoin)-ConstantExpr-Filter-TableFullScan-ConstantExpr",
3434
"Operators": [
3535
{
3636
"Inputs": [
37-
{
38-
"InternalOperatorId": 1
39-
},
40-
{
41-
"InternalOperatorId": 3
42-
},
4337
{
4438
"InternalOperatorId": 1
4539
},
@@ -90,23 +84,21 @@
9084
"Inputs": [
9185
{
9286
"InternalOperatorId": 2
87+
},
88+
{
89+
"InternalOperatorId": 6
9390
}
9491
],
95-
"Name": "Aggregate"
96-
},
97-
{
98-
"Inputs": [],
99-
"Name": "ToFlow",
100-
"ToFlow": "precompute_0_0"
92+
"Name": "CrossJoin"
10193
},
10294
{
10395
"Condition": "thousand = Sum0",
10496
"Inputs": [
10597
{
106-
"InternalOperatorId": 5
98+
"InternalOperatorId": 4
10799
},
108100
{
109-
"InternalOperatorId": 4
101+
"InternalOperatorId": 3
110102
}
111103
],
112104
"Name": "InnerJoin (MapJoin)"
@@ -119,7 +111,7 @@
119111
{
120112
"Inputs": [
121113
{
122-
"InternalOperatorId": 6
114+
"InternalOperatorId": 5
123115
}
124116
],
125117
"Name": "Filter",
@@ -137,6 +129,11 @@
137129
"unique2 (-\u221e, +\u221e)"
138130
],
139131
"Table": "postgres_jointest/join0.test_plan/tenk1"
132+
},
133+
{
134+
"Inputs": [],
135+
"Name": "ToFlow",
136+
"ToFlow": "precompute_0_0"
140137
}
141138
],
142139
"PlanNodeId": 6,

‎ydb/tests/functional/suite_tests/canondata/test_sql_logic.TestSQLLogic.test_sql_suite_plan-select1-1.test_/query_101.plan‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,10 @@
2929
"PlanNodeType": "Connection",
3030
"Plans": [
3131
{
32-
"Node Type": "TopSort-Aggregate",
32+
"Node Type": "TopSort-CrossJoin",
3333
"Operators": [
3434
{
3535
"Inputs": [
36-
{
37-
"InternalOperatorId": 1
38-
},
39-
{
40-
"ExternalPlanNodeId": 9
41-
},
4236
{
4337
"InternalOperatorId": 1
4438
}
@@ -49,11 +43,14 @@
4943
},
5044
{
5145
"Inputs": [
46+
{
47+
"ExternalPlanNodeId": 9
48+
},
5249
{
5350
"ExternalPlanNodeId": 7
5451
}
5552
],
56-
"Name": "Aggregate"
53+
"Name": "CrossJoin"
5754
}
5855
],
5956
"PlanNodeId": 10,

‎ydb/tests/functional/suite_tests/canondata/test_sql_logic.TestSQLLogic.test_sql_suite_plan-select1-1.test_/query_103.plan‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,10 @@
2929
"PlanNodeType": "Connection",
3030
"Plans": [
3131
{
32-
"Node Type": "TopSort-Aggregate",
32+
"Node Type": "TopSort-CrossJoin",
3333
"Operators": [
3434
{
3535
"Inputs": [
36-
{
37-
"InternalOperatorId": 1
38-
},
39-
{
40-
"ExternalPlanNodeId": 9
41-
},
4236
{
4337
"InternalOperatorId": 1
4438
}
@@ -49,11 +43,14 @@
4943
},
5044
{
5145
"Inputs": [
46+
{
47+
"ExternalPlanNodeId": 9
48+
},
5249
{
5350
"ExternalPlanNodeId": 7
5451
}
5552
],
56-
"Name": "Aggregate"
53+
"Name": "CrossJoin"
5754
}
5855
],
5956
"PlanNodeId": 10,

0 commit comments

Comments
 (0)