Skip to content

Commit f0f7d4b

Browse files
kardymondskardymonds
committed
YDB-2568 Enable match_recognize in ydb
1 parent 71d6ac0 commit f0f7d4b

File tree

23 files changed

+12796
-8180
lines changed

23 files changed

+12796
-8180
lines changed

‎ydb/core/kqp/compile_service/kqp_compile_actor.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
182182
Config->FeatureFlags = AppData(ctx)->FeatureFlags;
183183

184184
KqpHost = CreateKqpHost(Gateway, QueryId.Cluster, QueryId.Database, Config, ModuleResolverState->ModuleResolver,
185-
FederatedQuerySetup, UserToken, AppData(ctx)->FunctionRegistry, false, false, std::move(TempTablesState));
185+
FederatedQuerySetup, UserToken, QueryServiceConfig, AppData(ctx)->FunctionRegistry, false, false, std::move(TempTablesState));
186186

187187
IKqpHost::TPrepareSettings prepareSettings;
188188
prepareSettings.DocumentApiRestricted = QueryId.Settings.DocumentApiRestricted;

‎ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <ydb/library/yql/core/services/mounts/yql_mounts.h>
99

1010
#include <library/cpp/protobuf/util/pb_io.h>
11+
#include <ydb/core/protos/config.pb.h>
1112

1213
namespace NKikimr {
1314
namespace NKqp {
@@ -28,7 +29,7 @@ NKqpProto::TKqpPhyTx BuildTxPlan(const TString& sql, TIntrusivePtr<IKqpGateway>
2829
IModuleResolver::TPtr moduleResolver;
2930
UNIT_ASSERT(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver));
3031

31-
auto qp = CreateKqpHost(gateway, cluster, "/Root", config, moduleResolver, NYql::IHTTPGateway::Make(), nullptr, nullptr, false, false, nullptr, actorSystem);
32+
auto qp = CreateKqpHost(gateway, cluster, "/Root", config, moduleResolver, NYql::IHTTPGateway::Make(), nullptr, NKikimrConfig::TQueryServiceConfig(), nullptr, false, false, nullptr, actorSystem);
3233
auto result = qp->SyncPrepareDataQuery(sql, IKqpHost::TPrepareSettings());
3334
result.Issues().PrintTo(Cerr);
3435
UNIT_ASSERT(result.Success());

‎ydb/core/kqp/host/kqp_host.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,9 +955,15 @@ class TKqpHost : public IKqpHost {
955955
TKqpHost(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, const TString& database,
956956
TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver,
957957
std::optional<TKqpFederatedQuerySetup> federatedQuerySetup, const TIntrusiveConstPtr<NACLib::TUserToken>& userToken,
958+
<<<<<<< HEAD
958959
const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges,
959960
bool isInternalCall, TKqpTempTablesState::TConstPtr tempTablesState = nullptr,
960961
NActors::TActorSystem* actorSystem = nullptr)
962+
=======
963+
const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges, bool isInternalCall,
964+
TKqpTempTablesState::TConstPtr tempTablesState = nullptr, NActors::TActorSystem* actorSystem = nullptr,
965+
NYql::TExprContext* ctx = nullptr, const NKikimrConfig::TQueryServiceConfig& queryServiceConfig = NKikimrConfig::TQueryServiceConfig())
966+
>>>>>>> 18b7d766e7... YDB-2568 Enable match_recognize in ydb (#6807)
961967
: Gateway(gateway)
962968
, Cluster(cluster)
963969
, ExprCtx(new TExprContext())
@@ -972,6 +978,7 @@ class TKqpHost : public IKqpHost {
972978
, FakeWorld(ExprCtx->NewWorld(TPosition()))
973979
, ExecuteCtx(MakeIntrusive<TExecuteContext>())
974980
, ActorSystem(actorSystem ? actorSystem : NActors::TActivationContext::ActorSystem())
981+
, QueryServiceConfig(queryServiceConfig)
975982
{
976983
if (funcRegistry) {
977984
FuncRegistry = funcRegistry;
@@ -1605,10 +1612,21 @@ class TKqpHost : public IKqpHost {
16051612
|| settingName == "Warning"
16061613
|| settingName == "UseBlocks"
16071614
|| settingName == "BlockEngine"
1615+
<<<<<<< HEAD
1616+
=======
1617+
|| settingName == "FilterPushdownOverJoinOptionalSide"
1618+
|| settingName == "DisableFilterPushdownOverJoinOptionalSide"
1619+
|| settingName == "RotateJoinTree"
1620+
|| settingName == "TimeOrderRecoverDelay"
1621+
|| settingName == "TimeOrderRecoverAhead"
1622+
|| settingName == "TimeOrderRecoverRowLimit"
1623+
|| settingName == "MatchRecognizeStream"
1624+
>>>>>>> 18b7d766e7... YDB-2568 Enable match_recognize in ydb (#6807)
16081625
;
16091626
};
16101627
auto configProvider = CreateConfigProvider(*TypesCtx, gatewaysConfig, {}, allowSettings);
16111628
TypesCtx->AddDataSource(ConfigProviderName, configProvider);
1629+
TypesCtx->MatchRecognize = QueryServiceConfig.GetEnableMatchRecognize();
16121630

16131631
YQL_ENSURE(TypesCtx->Initialize(*ExprCtx));
16141632

@@ -1701,6 +1719,7 @@ class TKqpHost : public IKqpHost {
17011719

17021720
TKqpTempTablesState::TConstPtr TempTablesState;
17031721
NActors::TActorSystem* ActorSystem = nullptr;
1722+
NKikimrConfig::TQueryServiceConfig QueryServiceConfig;
17041723
};
17051724

17061725
} // namespace
@@ -1721,6 +1740,7 @@ Ydb::Table::QueryStatsCollection::Mode GetStatsMode(NYql::EKikimrStatsMode stats
17211740
TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway,
17221741
const TString& cluster, const TString& database, TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver,
17231742
std::optional<TKqpFederatedQuerySetup> federatedQuerySetup, const TIntrusiveConstPtr<NACLib::TUserToken>& userToken,
1743+
const NKikimrConfig::TQueryServiceConfig& queryServiceConfig,
17241744
const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges, bool isInternalCall,
17251745
TKqpTempTablesState::TConstPtr tempTablesState, NActors::TActorSystem* actorSystem)
17261746
{

‎ydb/core/kqp/host/kqp_host.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,8 @@ class IKqpHost : public TThrRefBase {
109109

110110
TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway,
111111
const TString& cluster, const TString& database, NYql::TKikimrConfiguration::TPtr config, NYql::IModuleResolver::TPtr moduleResolver,
112-
std::optional<TKqpFederatedQuerySetup> federatedQuerySetup, const TIntrusiveConstPtr<NACLib::TUserToken>& userToken, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr,
112+
std::optional<TKqpFederatedQuerySetup> federatedQuerySetup, const TIntrusiveConstPtr<NACLib::TUserToken>& userToken,
113+
const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr,
113114
bool keepConfigChanges = false, bool isInternalCall = false, TKqpTempTablesState::TConstPtr tempTablesState = nullptr,
114115
NActors::TActorSystem* actorSystem = nullptr /*take from TLS by default*/);
115116

‎ydb/core/kqp/opt/logical/kqp_opt_log.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h>
77
#include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h>
88

9+
#include <ydb/library/yql/core/yql_opt_match_recognize.h>
910
#include <ydb/library/yql/core/yql_opt_utils.h>
1011
#include <ydb/library/yql/dq/opt/dq_opt_join.h>
1112
#include <ydb/library/yql/dq/opt/dq_opt_log.h>
@@ -54,7 +55,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
5455
AddHandler(0, &TKqlLookupTableBase::Match, HNDL(ApplyExtractMembersToLookupTable<false>));
5556
AddHandler(0, &TCoTop::Match, HNDL(TopSortOverExtend));
5657
AddHandler(0, &TCoTopSort::Match, HNDL(TopSortOverExtend));
57-
58+
AddHandler(0, &TCoMatchRecognize::Match, HNDL(MatchRecognize));
5859
AddHandler(1, &TCoFlatMap::Match, HNDL(LatePushExtractedPredicateToReadTable));
5960
AddHandler(1, &TCoTop::Match, HNDL(RewriteTopSortOverIndexRead));
6061
AddHandler(1, &TCoTopSort::Match, HNDL(RewriteTopSortOverIndexRead));
@@ -255,6 +256,14 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
255256
return output;
256257
}
257258

259+
TMaybeNode<TExprBase> MatchRecognize(TExprBase node, TExprContext& ctx) {
260+
auto output = ExpandMatchRecognize(node.Ptr(), ctx, TypesCtx);
261+
if (output) {
262+
DumpAppliedRule("MatchRecognize", node.Ptr(), output, ctx);
263+
}
264+
return output;
265+
}
266+
258267
template <bool IsGlobal>
259268
TMaybeNode<TExprBase> ApplyExtractMembersToReadTable(TExprBase node, TExprContext& ctx,
260269
const TGetParents& getParents)

‎ydb/core/kqp/session_actor/kqp_worker_actor.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ class TKqpWorkerActor : public TActorBootstrapped<TKqpWorkerActor> {
187187
Config->FeatureFlags = AppData(ctx)->FeatureFlags;
188188

189189
KqpHost = CreateKqpHost(Gateway, Settings.Cluster, Settings.Database, Config, ModuleResolverState->ModuleResolver,
190-
FederatedQuerySetup, QueryState->RequestEv->GetUserToken(), AppData(ctx)->FunctionRegistry, !Settings.LongSession, false);
190+
FederatedQuerySetup, QueryState->RequestEv->GetUserToken(), QueryServiceConfig, AppData(ctx)->FunctionRegistry, !Settings.LongSession, false);
191191

192192
auto& queryRequest = QueryState->RequestEv;
193193
QueryState->ProxyRequestId = proxyRequestId;

‎ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ TIntrusivePtr<IKqpHost> CreateKikimrQueryProcessor(TIntrusivePtr<IKqpGateway> ga
5555

5656
auto federatedQuerySetup = std::make_optional<TKqpFederatedQuerySetup>({NYql::IHTTPGateway::Make(), nullptr, nullptr, nullptr, {}, {}, {}});
5757
return NKqp::CreateKqpHost(gateway, cluster, "/Root", kikimrConfig, moduleResolver,
58-
federatedQuerySetup, nullptr, funcRegistry, funcRegistry, keepConfigChanges, nullptr, actorSystem);
58+
federatedQuerySetup, nullptr, NKikimrConfig::TQueryServiceConfig(), funcRegistry, funcRegistry, keepConfigChanges, nullptr, actorSystem);
5959
}
6060

6161
NYql::NNodes::TExprBase GetExpr(const TString& ast, NYql::TExprContext& ctx, NYql::IModuleResolver* moduleResolver) {

‎ydb/core/kqp/ut/yql/kqp_pragma_ut.cpp‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,93 @@ Y_UNIT_TEST_SUITE(KqpPragma) {
8484
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
8585
UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString());
8686
}
87+
88+
Y_UNIT_TEST(MatchRecognizeWithTimeOrderRecoverer) {
89+
TKikimrSettings settings;
90+
NKikimrConfig::TAppConfig appConfig;
91+
appConfig.MutableQueryServiceConfig()->SetEnableMatchRecognize(true);
92+
settings.SetAppConfig(appConfig);
93+
94+
TKikimrRunner kikimr(settings);
95+
NYdb::NScripting::TScriptingClient client(kikimr.GetDriver());
96+
97+
auto result = client.ExecuteYqlScript(R"(
98+
PRAGMA FeatureR010="prototype";
99+
100+
CREATE TABLE `/Root/NewTable` (
101+
dt Uint64,
102+
value String,
103+
PRIMARY KEY (dt)
104+
);
105+
COMMIT;
106+
107+
INSERT INTO `/Root/NewTable` (dt, value) VALUES
108+
(1, 'value1'), (2, 'value2'), (3, 'value3'), (4, 'value4');
109+
COMMIT;
110+
111+
SELECT * FROM (SELECT dt, value FROM `/Root/NewTable`)
112+
MATCH_RECOGNIZE(
113+
ORDER BY CAST(dt as Timestamp)
114+
MEASURES
115+
LAST(V1.dt) as v1,
116+
LAST(V4.dt) as v4
117+
ONE ROW PER MATCH
118+
PATTERN (V1 V* V4)
119+
DEFINE
120+
V1 as V1.value = "value1",
121+
V as True,
122+
V4 as V4.value = "value4"
123+
);
124+
)").GetValueSync();
125+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
126+
CompareYson(R"([
127+
[[1u];[4u]];
128+
])", FormatResultSetYson(result.GetResultSet(0)));
129+
}
130+
131+
Y_UNIT_TEST(MatchRecognizeWithoutTimeOrderRecoverer) {
132+
TKikimrSettings settings;
133+
NKikimrConfig::TAppConfig appConfig;
134+
appConfig.MutableQueryServiceConfig()->SetEnableMatchRecognize(true);
135+
settings.SetAppConfig(appConfig);
136+
137+
TKikimrRunner kikimr(settings);
138+
NYdb::NScripting::TScriptingClient client(kikimr.GetDriver());
139+
140+
auto result = client.ExecuteYqlScript(R"(
141+
PRAGMA FeatureR010="prototype";
142+
PRAGMA config.flags("MatchRecognizeStream", "disable");
143+
144+
CREATE TABLE `/Root/NewTable` (
145+
dt Uint64,
146+
value String,
147+
PRIMARY KEY (dt)
148+
);
149+
COMMIT;
150+
151+
INSERT INTO `/Root/NewTable` (dt, value) VALUES
152+
(1, 'value1'), (2, 'value2'), (3, 'value3'), (4, 'value4');
153+
COMMIT;
154+
155+
SELECT * FROM (SELECT dt, value FROM `/Root/NewTable`)
156+
MATCH_RECOGNIZE(
157+
ORDER BY CAST(dt as Timestamp)
158+
MEASURES
159+
LAST(V1.dt) as v1,
160+
LAST(V4.dt) as v4
161+
ONE ROW PER MATCH
162+
PATTERN (V1 V* V4)
163+
DEFINE
164+
V1 as V1.value = "value1",
165+
V as True,
166+
V4 as V4.value = "value4"
167+
);
168+
)").GetValueSync();
169+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
170+
CompareYson(R"([
171+
[[1u];[4u]];
172+
])", FormatResultSetYson(result.GetResultSet(0)));
173+
}
87174
}
88175

89176
} // namspace NKqp

‎ydb/core/protos/config.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1052,6 +1052,7 @@ message TQueryServiceConfig {
10521052
optional NYql.TGenericGatewayConfig Generic = 11;
10531053
optional TFinalizeScriptServiceConfig FinalizeScriptServiceConfig = 12;
10541054
optional uint64 ProgressStatsPeriodMs = 14 [default = 0]; // 0 = disabled
1055+
optional bool EnableMatchRecognize = 20 [default = false];
10551056
}
10561057

10571058
// Config describes immediate controls and allows

‎ydb/library/yql/core/yql_opt_match_recognize.cpp‎

Lines changed: 10 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ TExprNode::TPtr ExpandMatchRecognize(const TExprNode::TPtr& node, TExprContext&
146146
ExtractSortKeyAndOrder(pos, sortTraits, sortKey, sortOrder, ctx);
147147
TExprNode::TPtr result;
148148
if (isStreaming) {
149-
YQL_ENSURE(sortOrder->ChildrenSize() == 1, "Expect ORDER BY timestamp for MATCH_RECOGNIZE on streams");
149+
YQL_ENSURE(sortOrder->ChildrenSize() == 1, "Expect ORDER BY timestamp for MATCH_RECOGNIZE");
150150
const auto reordered = ctx.Builder(pos)
151151
.Lambda()
152152
.Param("partition")
@@ -216,37 +216,15 @@ TExprNode::TPtr ExpandMatchRecognize(const TExprNode::TPtr& node, TExprContext&
216216
.Seal()
217217
.Build();
218218
} else { //non-streaming
219-
if (partitionColumns->ChildrenSize() != 0) {
220-
result = ctx.Builder(pos)
221-
.Callable("PartitionsByKeys")
222-
.Add(0, input)
223-
.Add(1, partitionKeySelector)
224-
.Add(2, sortOrder)
225-
.Add(3, sortKey)
226-
.Add(4, matchRecognize)
227-
.Seal()
228-
.Build();
229-
} else {
230-
if (sortOrder->IsCallable("Void")) {
231-
result = ctx.Builder(pos)
232-
.Apply(matchRecognize)
233-
.With(0, input)
234-
.Seal()
235-
.Build();;
236-
} else {
237-
result = ctx.Builder(pos)
238-
.Apply(matchRecognize)
239-
.With(0)
240-
.Callable("Sort")
241-
.Add(0, input)
242-
.Add(1, sortOrder)
243-
.Add(2, sortKey)
244-
.Seal()
245-
.Done()
246-
.Seal()
247-
.Build();
248-
}
249-
}
219+
result = ctx.Builder(pos)
220+
.Callable("PartitionsByKeys")
221+
.Add(0, input)
222+
.Add(1, partitionKeySelector)
223+
.Add(2, sortOrder)
224+
.Add(3, sortKey)
225+
.Add(4, matchRecognize)
226+
.Seal()
227+
.Build();
250228
}
251229
YQL_CLOG(INFO, Core) << "Expanded MatchRecognize";
252230
return result;

0 commit comments

Comments
 (0)