Skip to content

Commit c8eca1f

Browse files
shnikdshnikd
authored
Merge 0cccdfa into e32508c 
2 parents e32508c + 0cccdfa commit c8eca1f

16 files changed

+337
-105
lines changed

‎ydb/core/kqp/executer_actor/kqp_table_resolver.cpp

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,9 @@
44
#include <ydb/core/base/cputime.h>
55
#include <ydb/core/base/path.h>
66
#include <ydb/core/kqp/executer_actor/kqp_executer.h>
7+
#include <ydb/core/sys_view/common/schema.h>
78
#include <ydb/library/actors/core/actor_bootstrapped.h>
89

9-
10-
1110
namespace NKikimr::NKqp {
1211

1312
using namespace NActors;
@@ -35,7 +34,8 @@ class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
3534
, TxId(txId)
3635
, UserToken(userToken)
3736
, Transactions(transactions)
38-
, TasksGraph(tasksGraph) {}
37+
, TasksGraph(tasksGraph)
38+
, SystemViewRewrittenResolver(NSysView::CreateSystemViewRewrittenResolver()) {}
3939

4040
void Bootstrap() {
4141
ResolveKeys();
@@ -173,6 +173,10 @@ class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
173173

174174
stageInfo.Meta.ShardKey = ExtractKey(stageInfo.Meta.TableId, stageInfo.Meta.TableConstInfo, operation);
175175

176+
if (SystemViewRewrittenResolver->IsSystemView(stageInfo.Meta.TableId.SysViewInfo)) {
177+
continue;
178+
}
179+
176180
if (stageInfo.Meta.TableKind == ETableKind::Olap) {
177181
if (TableRequestIds.find(stageInfo.Meta.TableId) == TableRequestIds.end()) {
178182
auto& entry = requestNavigate->ResultSet.emplace_back();
@@ -276,6 +280,8 @@ class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
276280
bool ShouldTerminate = false;
277281
TMaybe<ui32> GotUnexpectedEvent;
278282
TDuration CpuTime;
283+
284+
THolder<NSysView::ISystemViewResolver> SystemViewRewrittenResolver;
279285
};
280286

281287
} // anonymous namespace

‎ydb/core/kqp/executer_actor/ya.make

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ SRCS(
1818
)
1919

2020
PEERDIR(
21-
ydb/library/actors/core
2221
library/cpp/containers/absl_flat_hash
2322
ydb/core/actorlib_impl
2423
ydb/core/base
@@ -28,20 +27,22 @@ PEERDIR(
2827
ydb/core/kqp/compute_actor
2928
ydb/core/kqp/executer_actor/shards_resolver
3029
ydb/core/kqp/federated_query
30+
ydb/core/kqp/gateway/local_rpc
3131
ydb/core/kqp/query_compiler
3232
ydb/core/kqp/rm_service
3333
ydb/core/kqp/topics
34-
ydb/core/kqp/gateway/local_rpc
3534
ydb/core/protos
35+
ydb/core/sys_view/common
3636
ydb/core/tx/long_tx_service/public
3737
ydb/core/ydb_convert
38-
ydb/services/metadata/abstract
38+
ydb/library/actors/core
3939
ydb/library/mkql_proto
4040
ydb/library/mkql_proto/protos
4141
ydb/library/yql/dq/actors/compute
4242
ydb/library/yql/dq/runtime
4343
ydb/library/yql/dq/tasks
4444
ydb/library/yql/providers/common/http_gateway
45+
ydb/services/metadata/abstract
4546
)
4647

4748
GENERATE_ENUM_SERIALIZATION(

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,20 @@ void TKqpTableMetadataLoader::OnLoadedTableMetadata(TTableMetadataResult& loadTa
573573
}
574574
}
575575

576+
NThreading::TFuture<NYql::IKikimrGateway::TTableMetadataResult> TKqpTableMetadataLoader::LoadSysViewRewrittenMetadata(
577+
const NSysView::ISystemViewResolver::TSystemViewPath& sysViewPath, const TString& cluster, const TString& table
578+
) {
579+
TNavigate::TEntry entry;
580+
581+
auto schema = SystemViewRewrittenResolver->GetSystemViewSchema(sysViewPath.ViewName, NSysView::ISystemViewResolver::ETarget::Domain);
582+
entry.Kind = TNavigate::KindTable;
583+
entry.Columns = std::move(schema->Columns);
584+
entry.TableId = TTableId(TSysTables::SysSchemeShard, 0, sysViewPath.ViewName);
585+
586+
auto result = GetTableMetadataResult(entry, cluster, table);
587+
588+
return MakeFuture(result);
589+
}
576590

577591
NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMetadata(const TString& cluster, const TString& table,
578592
const NYql::IKikimrGateway::TLoadTableMetadataSettings& settings, const TString& database,
@@ -582,7 +596,14 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta
582596

583597
auto ptr = weak_from_base();
584598
try {
585-
auto tableMetaFuture = LoadTableMetadataCache(cluster, table, settings, database, userToken);
599+
NThreading::TFuture<TTableMetadataResult> tableMetaFuture;
600+
601+
NSysView::ISystemViewResolver::TSystemViewPath sysViewPath;
602+
if (settings.SysViewRewritten_ && SystemViewRewrittenResolver->IsSystemViewPath(SplitPath(table), sysViewPath)) {
603+
tableMetaFuture = LoadSysViewRewrittenMetadata(sysViewPath, cluster, table);
604+
} else {
605+
tableMetaFuture = LoadTableMetadataCache(cluster, table, settings, database, userToken);
606+
}
586607
return tableMetaFuture.Apply([ptr, database, userToken](const TFuture<TTableMetadataResult>& future) mutable {
587608
try {
588609
auto result = future.GetValue();
@@ -947,7 +968,8 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta
947968
catch (yexception& e) {
948969
promise.SetValue(ResultFromException<TResult>(e));
949970
}
950-
});
971+
}
972+
);
951973

952974
// Create an apply for the future that will fetch table statistics and save it in the metadata
953975
// This method will only run if cost based optimization is enabled

‎ydb/core/kqp/gateway/kqp_metadata_loader.h

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
#include <ydb/core/kqp/common/simple/temp_tables.h>
44
#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
5+
#include <ydb/core/kqp/provider/yql_kikimr_settings.h>
56
#include <ydb/core/scheme/scheme_tabledefs.h>
7+
#include <ydb/core/sys_view/common/schema.h>
68
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
7-
#include <ydb/core/kqp/provider/yql_kikimr_settings.h>
9+
810
#include <library/cpp/threading/future/core/future.h>
911

1012
#include <util/system/mutex.h>
@@ -29,6 +31,7 @@ class TKqpTableMetadataLoader : public NYql::IKikimrGateway::IKqpTableMetadataLo
2931
, ActorSystem(actorSystem)
3032
, Config(config)
3133
, TempTablesState(std::move(tempTablesState))
34+
, SystemViewRewrittenResolver(NSysView::CreateSystemViewRewrittenResolver())
3235
{}
3336

3437
NThreading::TFuture<NYql::IKikimrGateway::TTableMetadataResult> LoadTableMetadata(
@@ -61,13 +64,18 @@ class TKqpTableMetadataLoader : public NYql::IKikimrGateway::IKqpTableMetadataLo
6164

6265
void OnLoadedTableMetadata(NYql::IKikimrGateway::TTableMetadataResult& loadTableMetadataResult);
6366

67+
NThreading::TFuture<NYql::IKikimrGateway::TTableMetadataResult> LoadSysViewRewrittenMetadata(
68+
const NSysView::ISystemViewResolver::TSystemViewPath& sysViewPath, const TString& cluster, const TString& table
69+
);
70+
6471
const TString Cluster;
6572
TVector<NKikimrKqp::TKqpTableMetadataProto> CollectedSchemeData;
6673
TMutex Lock;
6774
bool NeedCollectSchemeData;
6875
TActorSystem* ActorSystem;
6976
NYql::TKikimrConfiguration::TPtr Config;
7077
TKqpTempTablesState::TConstPtr TempTablesState;
78+
THolder<NSysView::ISystemViewResolver> SystemViewRewrittenResolver;
7179
};
7280

7381
} // namespace NKikimr::NKqp

‎ydb/core/kqp/gateway/ya.make

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,22 @@ SRCS(
77
)
88

99
PEERDIR(
10-
ydb/library/actors/core
1110
ydb/core/actorlib_impl
1211
ydb/core/base
1312
ydb/core/kqp/common
14-
ydb/core/kqp/provider
15-
ydb/core/kqp/query_data
1613
ydb/core/kqp/gateway/actors
17-
ydb/core/kqp/gateway/behaviour/tablestore
18-
ydb/core/kqp/gateway/behaviour/table
1914
ydb/core/kqp/gateway/behaviour/external_data_source
2015
ydb/core/kqp/gateway/behaviour/resource_pool
2116
ydb/core/kqp/gateway/behaviour/resource_pool_classifier
17+
ydb/core/kqp/gateway/behaviour/table
18+
ydb/core/kqp/gateway/behaviour/tablestore
2219
ydb/core/kqp/gateway/behaviour/view
2320
ydb/core/kqp/gateway/utils
24-
ydb/core/statistics/service
21+
ydb/core/kqp/provider
22+
ydb/core/kqp/query_data
23+
ydb/core/statistics/service
24+
ydb/core/sys_view/common
25+
ydb/library/actors/core
2526
yql/essentials/providers/result/expr_nodes
2627
)
2728

‎ydb/core/kqp/provider/yql_kikimr_datasource.cpp

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,23 +102,48 @@ class TKiSourceIntentDeterminationTransformer: public TKiSourceVisitorTransforme
102102

103103
private:
104104
TStatus HandleKiRead(TKiReadBase node, TExprContext& ctx) override {
105+
bool sysViewRewritten = false;
106+
TExprBase currentNode(node);
107+
if (auto maybeReadTable = currentNode.Maybe<TKiReadTable>()) {
108+
auto readTable = maybeReadTable.Cast();
109+
for (auto setting : readTable.Settings()) {
110+
auto name = setting.Name().Value();
111+
if (name == "sysViewRewritten") {
112+
sysViewRewritten = true;
113+
}
114+
}
115+
}
116+
105117
auto cluster = node.DataSource().Cluster();
106118
TKikimrKey key(ctx);
107119
if (!key.Extract(node.TableKey().Ref())) {
108120
return TStatus::Error;
109121
}
110122

111-
return HandleKey(cluster, key);
123+
return HandleKey(cluster, key, sysViewRewritten);
112124
}
113125

114126
TStatus HandleRead(TExprBase node, TExprContext& ctx) override {
127+
bool sysViewRewritten = false;
128+
if (auto maybeRead = node.Maybe<TCoRead>()) {
129+
auto read = maybeRead.Cast();
130+
for (auto arg : read.FreeArgs()) {
131+
if (auto maybeTuple = arg.Maybe<TCoNameValueTuple>()) {
132+
auto tuple = maybeTuple.Cast();
133+
if (tuple.Ref().Child(0)->Content() == "sysViewRewritten") {
134+
sysViewRewritten = true;
135+
}
136+
}
137+
}
138+
}
139+
115140
auto cluster = node.Ref().Child(1)->Child(1)->Content();
116141
TKikimrKey key(ctx);
117142
if (!key.Extract(*node.Ref().Child(2))) {
118143
return TStatus::Error;
119144
}
120145

121-
return HandleKey(cluster, key);
146+
return HandleKey(cluster, key, sysViewRewritten);
122147
}
123148

124149
TStatus HandleLength(TExprBase node, TExprContext& ctx) override {
@@ -134,12 +159,12 @@ class TKiSourceIntentDeterminationTransformer: public TKiSourceVisitorTransforme
134159
}
135160

136161
private:
137-
TStatus HandleKey(const TStringBuf& cluster, const TKikimrKey& key) {
162+
TStatus HandleKey(const TStringBuf& cluster, const TKikimrKey& key, bool sysViewRewritten = false) {
138163
switch (key.GetKeyType()) {
139164
case TKikimrKey::Type::Table:
140165
case TKikimrKey::Type::TableScheme: {
141166
auto& table = SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(),
142-
key.GetTablePath());
167+
key.GetTablePath(), ETableType::Table, sysViewRewritten);
143168

144169
if (key.GetKeyType() == TKikimrKey::Type::TableScheme) {
145170
table.RequireStats();
@@ -238,6 +263,7 @@ class TKiSourceLoadTableMetadataTransformer : public TGraphTransformerBase {
238263
.WithAuthInfo(table.GetNeedAuthInfo())
239264
.WithExternalSourceFactory(ExternalSourceFactory)
240265
.WithReadAttributes(readAttrs ? std::move(*readAttrs) : THashMap<TString, TString>{})
266+
.WithSysViewRewritten(table.GetSysViewRewritten())
241267
);
242268

243269
futures.push_back(future.Apply([result, queryType]

‎ydb/core/kqp/provider/yql_kikimr_gateway.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1092,12 +1092,18 @@ class IKikimrGateway : public TThrRefBase {
10921092
return *this;
10931093
}
10941094

1095+
TLoadTableMetadataSettings& WithSysViewRewritten(bool enable) {
1096+
SysViewRewritten_ = enable;
1097+
return *this;
1098+
}
1099+
10951100
NKikimr::NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory;
10961101
THashMap<TString, TString> ReadAttributes;
10971102
bool RequestStats_ = false;
10981103
bool WithPrivateTables_ = false;
10991104
bool WithExternalDatasources_ = false;
11001105
bool RequestAuthInfo_ = true;
1106+
bool SysViewRewritten_ = false;
11011107
};
11021108

11031109
class IKqpTableMetadataLoader : public std::enable_shared_from_this<IKqpTableMetadataLoader> {

‎ydb/core/kqp/provider/yql_kikimr_opt_build.cpp

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -972,14 +972,19 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
972972
TKikimrKey key(ctx);
973973
YQL_ENSURE(key.Extract(content.TableKey().Ref()));
974974

975-
auto showCreateValue = Build<TCoNameValueTuple>(ctx, node.Pos())
975+
auto sysViewRewrittenValue = Build<TCoNameValueTuple>(ctx, node.Pos())
976976
.Name()
977-
.Build("showCreateTable")
977+
.Build("sysViewRewritten")
978978
.Value<TCoAtom>()
979979
.Value(key.GetTablePath())
980980
.Build()
981981
.Done();
982982

983+
auto showCreateTableValue = Build<TCoNameValueTuple>(ctx, node.Pos())
984+
.Name()
985+
.Build("showCreateTable")
986+
.Done();
987+
983988
auto showCreateTableRead = Build<TCoRead>(ctx, node.Pos())
984989
.World<TCoWorld>().Build()
985990
.DataSource<TCoDataSource>()
@@ -992,7 +997,8 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
992997
.Add(newKey)
993998
.Add(ctx.NewCallable(node.Pos(), "Void", {}))
994999
.Add(ctx.NewList(node.Pos(), {}))
995-
.Add(showCreateValue)
1000+
.Add(sysViewRewrittenValue)
1001+
.Add(showCreateTableValue)
9961002
.Build()
9971003
.Done().Ptr();
9981004

@@ -1009,14 +1015,21 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
10091015
auto right = rightMaybe.Cast();
10101016
if (auto maybeRead = right.Input().Maybe<TCoRead>()) {
10111017
auto read = maybeRead.Cast();
1018+
bool isSysViewRewritten = false;
1019+
bool isShowCreateTable = false;
10121020
for (auto arg : read.FreeArgs()) {
10131021
if (auto tuple = arg.Maybe<TCoNameValueTuple>()) {
10141022
auto name = tuple.Cast().Name().Value();
1015-
if (name == "showCreateTable") {
1016-
showCreateTableRightReplaces[input.Get()] = nullptr;
1023+
if (name == "sysViewRewritten") {
1024+
isSysViewRewritten = true;
1025+
} else if (name == "showCreateTable") {
1026+
isShowCreateTable = true;
10171027
}
10181028
}
10191029
}
1030+
if (isShowCreateTable && isSysViewRewritten) {
1031+
showCreateTableRightReplaces[input.Get()] = nullptr;
1032+
}
10201033
}
10211034
}
10221035
return true;
@@ -1030,7 +1043,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
10301043
for (auto arg : read.FreeArgs()) {
10311044
if (auto tuple = arg.Maybe<TCoNameValueTuple>()) {
10321045
auto name = tuple.Cast().Name().Value();
1033-
if (name == "showCreateTable") {
1046+
if (name == "sysViewRewritten") {
10341047
tablePath = tuple.Cast().Value().Cast().Cast<TCoAtom>().StringValue();
10351048
}
10361049
}
@@ -1109,7 +1122,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
11091122
.Repeat(TExprStep::LoadTablesMetadata)
11101123
.Repeat(TExprStep::RewriteIO);
11111124

1112-
return ctx.ReplaceNodes(std::move(resNode.Ptr()), showCreateTableRightReplaces);;
1125+
return ctx.ReplaceNodes(std::move(resNode.Ptr()), showCreateTableRightReplaces);
11131126
}
11141127

11151128
TKiExploreTxResults txExplore;

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,8 @@ const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TStrin
172172
return nullptr;
173173
}
174174

175-
TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table, ETableType tableType) {
175+
TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database,
176+
const TString& table, ETableType tableType, bool sysViewRewritten) {
176177
auto tablePath = table;
177178
if (TempTablesState) {
178179
auto tempTableInfoIt = TempTablesState->FindInfo(table, true);
@@ -192,6 +193,8 @@ TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster
192193
}
193194
desc.SetTableType(tableType);
194195

196+
desc.SetSysViewRewritten(sysViewRewritten);
197+
195198
return desc;
196199
}
197200

0 commit comments

Comments
 (0)