Skip to content

Update system view show_create #15825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 18, 2025
Merged

Update system view show_create #15825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b75a212
Initial commit
shnikd Mar 17, 2025
4ad4b42
Fixes
shnikd Mar 17, 2025
9a774c0
Fixes
shnikd Mar 18, 2025
0cccdfa
Fixes
shnikd Mar 18, 2025
b4ed4ef
Fixes
shnikd Mar 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions ydb/core/kqp/executer_actor/kqp_table_resolver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@
#include <ydb/core/base/cputime.h>
#include <ydb/core/base/path.h>
#include <ydb/core/kqp/executer_actor/kqp_executer.h>
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/library/actors/core/actor_bootstrapped.h>



namespace NKikimr::NKqp {

using namespace NActors;
Expand Down Expand Up @@ -35,7 +34,8 @@ class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
, TxId(txId)
, UserToken(userToken)
, Transactions(transactions)
, TasksGraph(tasksGraph) {}
, TasksGraph(tasksGraph)
, SystemViewRewrittenResolver(NSysView::CreateSystemViewRewrittenResolver()) {}

void Bootstrap() {
ResolveKeys();
Expand Down Expand Up @@ -173,6 +173,10 @@ class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {

stageInfo.Meta.ShardKey = ExtractKey(stageInfo.Meta.TableId, stageInfo.Meta.TableConstInfo, operation);

if (SystemViewRewrittenResolver->IsSystemView(stageInfo.Meta.TableId.SysViewInfo)) {
continue;
}

if (stageInfo.Meta.TableKind == ETableKind::Olap) {
if (TableRequestIds.find(stageInfo.Meta.TableId) == TableRequestIds.end()) {
auto& entry = requestNavigate->ResultSet.emplace_back();
Expand Down Expand Up @@ -276,6 +280,8 @@ class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
bool ShouldTerminate = false;
TMaybe<ui32> GotUnexpectedEvent;
TDuration CpuTime;

THolder<NSysView::ISystemViewResolver> SystemViewRewrittenResolver;
};

} // anonymous namespace
Expand Down
7 changes: 4 additions & 3 deletions ydb/core/kqp/executer_actor/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ SRCS(
)

PEERDIR(
ydb/library/actors/core
library/cpp/containers/absl_flat_hash
ydb/core/actorlib_impl
ydb/core/base
Expand All @@ -28,20 +27,22 @@ PEERDIR(
ydb/core/kqp/compute_actor
ydb/core/kqp/executer_actor/shards_resolver
ydb/core/kqp/federated_query
ydb/core/kqp/gateway/local_rpc
ydb/core/kqp/query_compiler
ydb/core/kqp/rm_service
ydb/core/kqp/topics
ydb/core/kqp/gateway/local_rpc
ydb/core/protos
ydb/core/sys_view/common
ydb/core/tx/long_tx_service/public
ydb/core/ydb_convert
ydb/services/metadata/abstract
ydb/library/actors/core
ydb/library/mkql_proto
ydb/library/mkql_proto/protos
ydb/library/yql/dq/actors/compute
ydb/library/yql/dq/runtime
ydb/library/yql/dq/tasks
ydb/library/yql/providers/common/http_gateway
ydb/services/metadata/abstract
)

GENERATE_ENUM_SERIALIZATION(
Expand Down
26 changes: 24 additions & 2 deletions ydb/core/kqp/gateway/kqp_metadata_loader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,20 @@ void TKqpTableMetadataLoader::OnLoadedTableMetadata(TTableMetadataResult& loadTa
}
}

NThreading::TFuture<NYql::IKikimrGateway::TTableMetadataResult> TKqpTableMetadataLoader::LoadSysViewRewrittenMetadata(
const NSysView::ISystemViewResolver::TSystemViewPath& sysViewPath, const TString& cluster, const TString& table
) {
TNavigate::TEntry entry;

auto schema = SystemViewRewrittenResolver->GetSystemViewSchema(sysViewPath.ViewName, NSysView::ISystemViewResolver::ETarget::Domain);
entry.Kind = TNavigate::KindTable;
entry.Columns = std::move(schema->Columns);
entry.TableId = TTableId(TSysTables::SysSchemeShard, 0, sysViewPath.ViewName);

auto result = GetTableMetadataResult(entry, cluster, table);

return MakeFuture(result);
}

NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMetadata(const TString& cluster, const TString& table,
const NYql::IKikimrGateway::TLoadTableMetadataSettings& settings, const TString& database,
Expand All @@ -582,7 +596,14 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta

auto ptr = weak_from_base();
try {
auto tableMetaFuture = LoadTableMetadataCache(cluster, table, settings, database, userToken);
NThreading::TFuture<TTableMetadataResult> tableMetaFuture;

NSysView::ISystemViewResolver::TSystemViewPath sysViewPath;
if (settings.SysViewRewritten_ && SystemViewRewrittenResolver->IsSystemViewPath(SplitPath(table), sysViewPath)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

а QueryReplay будет поддерживать Resolve такой таблицы?

tableMetaFuture = LoadSysViewRewrittenMetadata(sysViewPath, cluster, table);
} else {
tableMetaFuture = LoadTableMetadataCache(cluster, table, settings, database, userToken);
}
return tableMetaFuture.Apply([ptr, database, userToken](const TFuture<TTableMetadataResult>& future) mutable {
try {
auto result = future.GetValue();
Expand Down Expand Up @@ -947,7 +968,8 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta
catch (yexception& e) {
promise.SetValue(ResultFromException<TResult>(e));
}
});
}
);

// Create an apply for the future that will fetch table statistics and save it in the metadata
// This method will only run if cost based optimization is enabled
Expand Down
10 changes: 9 additions & 1 deletion ydb/core/kqp/gateway/kqp_metadata_loader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

#include <ydb/core/kqp/common/simple/temp_tables.h>
#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
#include <ydb/core/kqp/provider/yql_kikimr_settings.h>
#include <ydb/core/scheme/scheme_tabledefs.h>
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/core/kqp/provider/yql_kikimr_settings.h>

#include <library/cpp/threading/future/core/future.h>

#include <util/system/mutex.h>
Expand All @@ -29,6 +31,7 @@ class TKqpTableMetadataLoader : public NYql::IKikimrGateway::IKqpTableMetadataLo
, ActorSystem(actorSystem)
, Config(config)
, TempTablesState(std::move(tempTablesState))
, SystemViewRewrittenResolver(NSysView::CreateSystemViewRewrittenResolver())
{}

NThreading::TFuture<NYql::IKikimrGateway::TTableMetadataResult> LoadTableMetadata(
Expand Down Expand Up @@ -61,13 +64,18 @@ class TKqpTableMetadataLoader : public NYql::IKikimrGateway::IKqpTableMetadataLo

void OnLoadedTableMetadata(NYql::IKikimrGateway::TTableMetadataResult& loadTableMetadataResult);

NThreading::TFuture<NYql::IKikimrGateway::TTableMetadataResult> LoadSysViewRewrittenMetadata(
const NSysView::ISystemViewResolver::TSystemViewPath& sysViewPath, const TString& cluster, const TString& table
);

const TString Cluster;
TVector<NKikimrKqp::TKqpTableMetadataProto> CollectedSchemeData;
TMutex Lock;
bool NeedCollectSchemeData;
TActorSystem* ActorSystem;
NYql::TKikimrConfiguration::TPtr Config;
TKqpTempTablesState::TConstPtr TempTablesState;
THolder<NSysView::ISystemViewResolver> SystemViewRewrittenResolver;
};

} // namespace NKikimr::NKqp
13 changes: 7 additions & 6 deletions ydb/core/kqp/gateway/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,22 @@ SRCS(
)

PEERDIR(
ydb/library/actors/core
ydb/core/actorlib_impl
ydb/core/base
ydb/core/kqp/common
ydb/core/kqp/provider
ydb/core/kqp/query_data
ydb/core/kqp/gateway/actors
ydb/core/kqp/gateway/behaviour/tablestore
ydb/core/kqp/gateway/behaviour/table
ydb/core/kqp/gateway/behaviour/external_data_source
ydb/core/kqp/gateway/behaviour/resource_pool
ydb/core/kqp/gateway/behaviour/resource_pool_classifier
ydb/core/kqp/gateway/behaviour/table
ydb/core/kqp/gateway/behaviour/tablestore
ydb/core/kqp/gateway/behaviour/view
ydb/core/kqp/gateway/utils
ydb/core/statistics/service
ydb/core/kqp/provider
ydb/core/kqp/query_data
ydb/core/statistics/service
ydb/core/sys_view/common
ydb/library/actors/core
yql/essentials/providers/result/expr_nodes
)

Expand Down
34 changes: 30 additions & 4 deletions ydb/core/kqp/provider/yql_kikimr_datasource.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,23 +102,48 @@ class TKiSourceIntentDeterminationTransformer: public TKiSourceVisitorTransforme

private:
TStatus HandleKiRead(TKiReadBase node, TExprContext& ctx) override {
bool sysViewRewritten = false;
TExprBase currentNode(node);
if (auto maybeReadTable = currentNode.Maybe<TKiReadTable>()) {
auto readTable = maybeReadTable.Cast();
for (auto setting : readTable.Settings()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

костыль какой то? зачем это нужно?

Copy link
Collaborator Author

@shnikd shnikd Mar 17, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Чтобы вывести настройку для таблицы, что это переписанный Show Create Table

auto name = setting.Name().Value();
if (name == "sysViewRewritten") {
sysViewRewritten = true;
}
}
}

auto cluster = node.DataSource().Cluster();
TKikimrKey key(ctx);
if (!key.Extract(node.TableKey().Ref())) {
return TStatus::Error;
}

return HandleKey(cluster, key);
return HandleKey(cluster, key, sysViewRewritten);
}

TStatus HandleRead(TExprBase node, TExprContext& ctx) override {
bool sysViewRewritten = false;
if (auto maybeRead = node.Maybe<TCoRead>()) {
auto read = maybeRead.Cast();
for (auto arg : read.FreeArgs()) {
if (auto maybeTuple = arg.Maybe<TCoNameValueTuple>()) {
auto tuple = maybeTuple.Cast();
if (tuple.Ref().Child(0)->Content() == "sysViewRewritten") {
sysViewRewritten = true;
}
}
}
}

auto cluster = node.Ref().Child(1)->Child(1)->Content();
TKikimrKey key(ctx);
if (!key.Extract(*node.Ref().Child(2))) {
return TStatus::Error;
}

return HandleKey(cluster, key);
return HandleKey(cluster, key, sysViewRewritten);
}

TStatus HandleLength(TExprBase node, TExprContext& ctx) override {
Expand All @@ -134,12 +159,12 @@ class TKiSourceIntentDeterminationTransformer: public TKiSourceVisitorTransforme
}

private:
TStatus HandleKey(const TStringBuf& cluster, const TKikimrKey& key) {
TStatus HandleKey(const TStringBuf& cluster, const TKikimrKey& key, bool sysViewRewritten = false) {
switch (key.GetKeyType()) {
case TKikimrKey::Type::Table:
case TKikimrKey::Type::TableScheme: {
auto& table = SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(),
key.GetTablePath());
key.GetTablePath(), ETableType::Table, sysViewRewritten);

if (key.GetKeyType() == TKikimrKey::Type::TableScheme) {
table.RequireStats();
Expand Down Expand Up @@ -238,6 +263,7 @@ class TKiSourceLoadTableMetadataTransformer : public TGraphTransformerBase {
.WithAuthInfo(table.GetNeedAuthInfo())
.WithExternalSourceFactory(ExternalSourceFactory)
.WithReadAttributes(readAttrs ? std::move(*readAttrs) : THashMap<TString, TString>{})
.WithSysViewRewritten(table.GetSysViewRewritten())
);

futures.push_back(future.Apply([result, queryType]
Expand Down
6 changes: 6 additions & 0 deletions ydb/core/kqp/provider/yql_kikimr_gateway.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1092,12 +1092,18 @@ class IKikimrGateway : public TThrRefBase {
return *this;
}

TLoadTableMetadataSettings& WithSysViewRewritten(bool enable) {
SysViewRewritten_ = enable;
return *this;
}

NKikimr::NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory;
THashMap<TString, TString> ReadAttributes;
bool RequestStats_ = false;
bool WithPrivateTables_ = false;
bool WithExternalDatasources_ = false;
bool RequestAuthInfo_ = true;
bool SysViewRewritten_ = false;
};

class IKqpTableMetadataLoader : public std::enable_shared_from_this<IKqpTableMetadataLoader> {
Expand Down
27 changes: 20 additions & 7 deletions ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -972,14 +972,19 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
TKikimrKey key(ctx);
YQL_ENSURE(key.Extract(content.TableKey().Ref()));

auto showCreateValue = Build<TCoNameValueTuple>(ctx, node.Pos())
auto sysViewRewrittenValue = Build<TCoNameValueTuple>(ctx, node.Pos())
.Name()
.Build("showCreateTable")
.Build("sysViewRewritten")
.Value<TCoAtom>()
.Value(key.GetTablePath())
.Build()
.Done();

auto showCreateTableValue = Build<TCoNameValueTuple>(ctx, node.Pos())
.Name()
.Build("showCreateTable")
.Done();

auto showCreateTableRead = Build<TCoRead>(ctx, node.Pos())
.World<TCoWorld>().Build()
.DataSource<TCoDataSource>()
Expand All @@ -992,7 +997,8 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
.Add(newKey)
.Add(ctx.NewCallable(node.Pos(), "Void", {}))
.Add(ctx.NewList(node.Pos(), {}))
.Add(showCreateValue)
.Add(sysViewRewrittenValue)
.Add(showCreateTableValue)
.Build()
.Done().Ptr();

Expand All @@ -1009,14 +1015,21 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
auto right = rightMaybe.Cast();
if (auto maybeRead = right.Input().Maybe<TCoRead>()) {
auto read = maybeRead.Cast();
bool isSysViewRewritten = false;
bool isShowCreateTable = false;
for (auto arg : read.FreeArgs()) {
if (auto tuple = arg.Maybe<TCoNameValueTuple>()) {
auto name = tuple.Cast().Name().Value();
if (name == "showCreateTable") {
showCreateTableRightReplaces[input.Get()] = nullptr;
if (name == "sysViewRewritten") {
isSysViewRewritten = true;
} else if (name == "showCreateTable") {
isShowCreateTable = true;
}
}
}
if (isShowCreateTable && isSysViewRewritten) {
showCreateTableRightReplaces[input.Get()] = nullptr;
}
}
}
return true;
Expand All @@ -1030,7 +1043,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
for (auto arg : read.FreeArgs()) {
if (auto tuple = arg.Maybe<TCoNameValueTuple>()) {
auto name = tuple.Cast().Name().Value();
if (name == "showCreateTable") {
if (name == "sysViewRewritten") {
tablePath = tuple.Cast().Value().Cast().Cast<TCoAtom>().StringValue();
}
}
Expand Down Expand Up @@ -1109,7 +1122,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, TExprContext& ctx, TStringBuf datab
.Repeat(TExprStep::LoadTablesMetadata)
.Repeat(TExprStep::RewriteIO);

return ctx.ReplaceNodes(std::move(resNode.Ptr()), showCreateTableRightReplaces);;
return ctx.ReplaceNodes(std::move(resNode.Ptr()), showCreateTableRightReplaces);
}

TKiExploreTxResults txExplore;
Expand Down
5 changes: 4 additions & 1 deletion ydb/core/kqp/provider/yql_kikimr_provider.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,8 @@ const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TStrin
return nullptr;
}

TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table, ETableType tableType) {
TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database,
const TString& table, ETableType tableType, bool sysViewRewritten) {
auto tablePath = table;
if (TempTablesState) {
auto tempTableInfoIt = TempTablesState->FindInfo(table, true);
Expand All @@ -192,6 +193,8 @@ TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster
}
desc.SetTableType(tableType);

desc.SetSysViewRewritten(sysViewRewritten);

return desc;
}

Expand Down
Loading
Loading