Skip to content

Commit 8a9d8c9

Browse files
ivanmorozov333ivanmorozov333
authored
Merge dd102b5 into 785a089 
2 parents 785a089 + dd102b5 commit 8a9d8c9

File tree

12 files changed

+272
-4
lines changed

12 files changed

+272
-4
lines changed

‎ydb/core/sys_view/common/schema.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,9 +217,11 @@ class TSystemViewResolver : public ISystemViewResolver {
217217
RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName);
218218
RegisterOlapStoreSystemView<Schema::PrimaryIndexPortionStats>(StorePrimaryIndexPortionStatsName);
219219
RegisterOlapStoreSystemView<Schema::PrimaryIndexGranuleStats>(StorePrimaryIndexGranuleStatsName);
220+
RegisterOlapStoreSystemView<Schema::PrimaryIndexOptimizerStats>(StorePrimaryIndexOptimizerStatsName);
220221
RegisterColumnTableSystemView<Schema::PrimaryIndexStats>(TablePrimaryIndexStatsName);
221222
RegisterColumnTableSystemView<Schema::PrimaryIndexPortionStats>(TablePrimaryIndexPortionStatsName);
222223
RegisterColumnTableSystemView<Schema::PrimaryIndexGranuleStats>(TablePrimaryIndexGranuleStatsName);
224+
RegisterColumnTableSystemView<Schema::PrimaryIndexOptimizerStats>(TablePrimaryIndexOptimizerStatsName);
223225

224226
RegisterSystemView<Schema::TopPartitions>(TopPartitions1MinuteName);
225227
RegisterSystemView<Schema::TopPartitions>(TopPartitions1HourName);

‎ydb/core/sys_view/common/schema.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,11 @@ constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute";
3434
constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats";
3535
constexpr TStringBuf StorePrimaryIndexPortionStatsName = "store_primary_index_portion_stats";
3636
constexpr TStringBuf StorePrimaryIndexGranuleStatsName = "store_primary_index_granule_stats";
37+
constexpr TStringBuf StorePrimaryIndexOptimizerStatsName = "store_primary_index_optimizer_stats";
3738
constexpr TStringBuf TablePrimaryIndexStatsName = "primary_index_stats";
3839
constexpr TStringBuf TablePrimaryIndexPortionStatsName = "primary_index_portion_stats";
3940
constexpr TStringBuf TablePrimaryIndexGranuleStatsName = "primary_index_granule_stats";
41+
constexpr TStringBuf TablePrimaryIndexOptimizerStatsName = "primary_index_optimizer_stats";
4042

4143
constexpr TStringBuf TopPartitions1MinuteName = "top_partitions_one_minute";
4244
constexpr TStringBuf TopPartitions1HourName = "top_partitions_one_hour";
@@ -557,6 +559,33 @@ struct Schema : NIceDb::Schema {
557559
>;
558560
};
559561

562+
struct PrimaryIndexOptimizerStats: Table<14> {
563+
struct PathId: Column<1, NScheme::NTypeIds::Uint64> {};
564+
struct TabletId: Column<2, NScheme::NTypeIds::Uint64> {};
565+
struct TaskId: Column<3, NScheme::NTypeIds::Uint64> {};
566+
struct HostName: Column<4, NScheme::NTypeIds::Utf8> {};
567+
struct NodeId: Column<5, NScheme::NTypeIds::Uint64> {};
568+
struct Start: Column<6, NScheme::NTypeIds::Utf8> {};
569+
struct Finish: Column<7, NScheme::NTypeIds::Utf8> {};
570+
struct Details: Column<8, NScheme::NTypeIds::Utf8> {};
571+
struct Category: Column<9, NScheme::NTypeIds::Uint64> {};
572+
struct Weight: Column<10, NScheme::NTypeIds::Int64> {};
573+
574+
using TKey = TableKey<PathId, TabletId, TaskId>;
575+
using TColumns = TableColumns<
576+
PathId,
577+
TabletId,
578+
TaskId,
579+
HostName,
580+
NodeId,
581+
Start,
582+
Finish,
583+
Details,
584+
Category,
585+
Weight
586+
>;
587+
};
588+
560589
};
561590

562591
bool MaybeSystemViewPath(const TVector<TString>& path);

‎ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ NKikimr::TConclusionStatus TMetadataFromStore::DoFillMetadata(const NColumnShard
2222
auto pathInfos = logsIndex->GetTables(fromPathId, toPathId);
2323
for (auto&& pathInfo : pathInfos) {
2424
if (pathIds.emplace(pathInfo->GetPathId()).second) {
25-
metadata->IndexGranules.emplace_back(NAbstract::TGranuleMetaView(*pathInfo, metadata->IsDescSorted()));
25+
metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted()));
2626
}
2727
}
2828
}
@@ -50,7 +50,7 @@ NKikimr::TConclusionStatus TMetadataFromTable::DoFillMetadata(const NColumnShard
5050
if (!pathInfo) {
5151
continue;
5252
}
53-
metadata->IndexGranules.emplace_back(NAbstract::TGranuleMetaView(*pathInfo, metadata->IsDescSorted()));
53+
metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted()));
5454
break;
5555
}
5656
}

‎ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#pragma once
2+
#include "granule_view.h"
23
#include <ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h>
34
#include <ydb/core/tx/columnshard/engines/reader/common/description.h>
45

@@ -7,19 +8,27 @@ namespace NKikimr::NOlap::NReader::NSysView::NAbstract {
78
class IMetadataFiller {
89
private:
910
virtual TConclusionStatus DoFillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const = 0;
11+
12+
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const {
13+
return NAbstract::TGranuleMetaView(granule, reverse);
14+
}
1015
public:
1116
virtual ~IMetadataFiller() = default;
1217

1318
TConclusionStatus FillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const {
1419
return DoFillMetadata(shard, metadata, read);
1520
}
21+
22+
NAbstract::TGranuleMetaView BuildGranuleView(const TGranuleMeta& granule, const bool reverse) const {
23+
return DoBuildGranuleView(granule, reverse);
24+
}
25+
1626
};
1727

1828
class TMetadataFromStore: public IMetadataFiller {
1929
protected:
2030
virtual TConclusionStatus DoFillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const override;
2131
public:
22-
2332
};
2433

2534
class TMetadataFromTable: public IMetadataFiller {

‎ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22
#include <ydb/core/tx/columnshard/engines/storage/granule.h>
3+
#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h>
34

45
namespace NKikimr::NOlap::NReader::NSysView::NAbstract {
56

@@ -8,6 +9,7 @@ class TGranuleMetaView {
89
using TPortions = std::deque<std::shared_ptr<TPortionInfo>>;
910
YDB_READONLY(ui64, PathId, 0);
1011
YDB_READONLY_DEF(TPortions, Portions);
12+
YDB_READONLY_DEF(std::vector<NStorageOptimizer::TTaskDescription>, OptimizerTasks);
1113
public:
1214
TGranuleMetaView(const TGranuleMeta& granule, const bool reverse)
1315
: PathId(granule.GetPathId())
@@ -26,6 +28,14 @@ class TGranuleMetaView {
2628
}
2729
}
2830

31+
void FillOptimizerTasks(const TGranuleMeta& granule, const bool reverse) {
32+
OptimizerTasks = granule.GetOptimizerTasksDescription();
33+
std::sort(OptimizerTasks.begin(), OptimizerTasks.end());
34+
if (reverse) {
35+
std::reverse(OptimizerTasks.begin(), OptimizerTasks.end());
36+
}
37+
}
38+
2939
bool operator<(const TGranuleMetaView& item) const {
3040
return PathId < item.PathId;
3141
}

‎ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#include "optimizer.h"
2+
#include <ydb/core/formats/arrow/switch/switch_type.h>
3+
#include <ydb/core/tx/columnshard/blobs_action/common/const.h>
4+
#include <ydb/core/tx/columnshard/engines/reader/abstract/read_context.h>
5+
#include <util/system/hostname.h>
6+
7+
namespace NKikimr::NOlap::NReader::NSysView::NOptimizer {
8+
9+
bool TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayBuilder>>& builders, NAbstract::TGranuleMetaView& granule) const {
10+
for (auto&& i : granule.GetOptimizerTasks()) {
11+
NArrow::Append<arrow::UInt64Type>(*builders[0], granule.GetPathId());
12+
NArrow::Append<arrow::UInt64Type>(*builders[1], ReadMetadata->TabletId);
13+
NArrow::Append<arrow::UInt64Type>(*builders[2], i.GetTaskId());
14+
NArrow::Append<arrow::StringType>(*builders[3], HostNameField);
15+
NArrow::Append<arrow::UInt64Type>(*builders[4], NActors::TActivationContext::AsActorContext().SelfID.NodeId());
16+
NArrow::Append<arrow::StringType>(*builders[5], arrow::util::string_view(i.GetStart().data(), i.GetStart().size()));
17+
NArrow::Append<arrow::StringType>(*builders[6], arrow::util::string_view(i.GetFinish().data(), i.GetFinish().size()));
18+
NArrow::Append<arrow::StringType>(*builders[7], arrow::util::string_view(i.GetDetails().data(), i.GetDetails().size()));
19+
NArrow::Append<arrow::UInt64Type>(*builders[8], i.GetWeightCategory());
20+
NArrow::Append<arrow::Int64Type>(*builders[9], i.GetWeight());
21+
}
22+
return false;
23+
}
24+
25+
std::unique_ptr<TScanIteratorBase> TReadStatsMetadata::StartScan(const std::shared_ptr<TReadContext>& readContext) const {
26+
return std::make_unique<TStatsIterator>(readContext->GetReadMetadataPtrVerifiedAs<TReadStatsMetadata>());
27+
}
28+
29+
std::vector<std::pair<TString, NKikimr::NScheme::TTypeInfo>> TReadStatsMetadata::GetKeyYqlSchema() const {
30+
return GetColumns(TStatsIterator::StatsSchema, TStatsIterator::StatsSchema.KeyColumns);
31+
}
32+
33+
std::shared_ptr<NAbstract::TReadStatsMetadata> TConstructor::BuildMetadata(const NColumnShard::TColumnShard* self, const TReadDescription& read) const {
34+
auto* index = self->GetIndexOptional();
35+
return std::make_shared<TReadStatsMetadata>(index ? index->CopyVersionedIndexPtr() : nullptr, self->TabletID(),
36+
IsReverse ? TReadMetadataBase::ESorting::DESC : TReadMetadataBase::ESorting::ASC,
37+
read.GetProgram(), index ? index->GetVersionedIndex().GetLastSchema() : nullptr, read.GetSnapshot());
38+
}
39+
40+
}

‎ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#pragma once
2+
#include <ydb/core/sys_view/common/schema.h>
3+
#include <ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h>
4+
#include <ydb/core/tx/columnshard/engines/reader/sys_view/constructor/constructor.h>
5+
#include <util/system/hostname.h>
6+
7+
namespace NKikimr::NOlap::NReader::NSysView::NOptimizer {
8+
9+
class TConstructor: public TStatScannerConstructor<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats> {
10+
private:
11+
using TBase = TStatScannerConstructor<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats>;
12+
protected:
13+
virtual std::shared_ptr<NAbstract::TReadStatsMetadata> BuildMetadata(const NColumnShard::TColumnShard* self, const TReadDescription& read) const override;
14+
15+
public:
16+
using TBase::TBase;
17+
};
18+
19+
struct TReadStatsMetadata: public NAbstract::TReadStatsMetadata {
20+
private:
21+
using TBase = NAbstract::TReadStatsMetadata;
22+
using TSysViewSchema = NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats;
23+
public:
24+
using TBase::TBase;
25+
26+
virtual std::unique_ptr<TScanIteratorBase> StartScan(const std::shared_ptr<TReadContext>& readContext) const override;
27+
virtual std::vector<std::pair<TString, NScheme::TTypeInfo>> GetKeyYqlSchema() const override;
28+
};
29+
30+
class TStatsIterator : public NAbstract::TStatsIterator<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats> {
31+
private:
32+
const std::string HostNameField = HostName();
33+
using TBase = NAbstract::TStatsIterator<NKikimr::NSysView::Schema::PrimaryIndexOptimizerStats>;
34+
virtual ui32 PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const override {
35+
return granule.GetOptimizerTasks().size();
36+
}
37+
virtual bool AppendStats(const std::vector<std::unique_ptr<arrow::ArrayBuilder>>& builders, NAbstract::TGranuleMetaView& granule) const override;
38+
public:
39+
using TBase::TBase;
40+
};
41+
42+
class TMetadataFromStore: public NAbstract::TMetadataFromStore {
43+
protected:
44+
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const override {
45+
NAbstract::TGranuleMetaView result(granule, reverse);
46+
result.FillOptimizerTasks(granule, reverse);
47+
return result;
48+
}
49+
public:
50+
51+
};
52+
53+
class TMetadataFromTable: public NAbstract::TMetadataFromTable {
54+
protected:
55+
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const override {
56+
NAbstract::TGranuleMetaView result(granule, reverse);
57+
result.FillOptimizerTasks(granule, reverse);
58+
return result;
59+
}
60+
public:
61+
62+
};
63+
64+
class TStoreSysViewPolicy: public NAbstract::ISysViewPolicy {
65+
protected:
66+
virtual std::unique_ptr<IScannerConstructor> DoCreateConstructor(const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override {
67+
return std::make_unique<TConstructor>(snapshot, itemsLimit, reverse);
68+
}
69+
virtual std::shared_ptr<NAbstract::IMetadataFiller> DoCreateMetadataFiller() const override {
70+
return std::make_shared<TMetadataFromStore>();
71+
}
72+
public:
73+
static const inline TFactory::TRegistrator<TStoreSysViewPolicy> Registrator = TFactory::TRegistrator<TStoreSysViewPolicy>(TString(::NKikimr::NSysView::StorePrimaryIndexOptimizerStatsName));
74+
75+
};
76+
77+
class TTableSysViewPolicy: public NAbstract::ISysViewPolicy {
78+
protected:
79+
virtual std::unique_ptr<IScannerConstructor> DoCreateConstructor(const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override {
80+
return std::make_unique<TConstructor>(snapshot, itemsLimit, reverse);
81+
}
82+
virtual std::shared_ptr<NAbstract::IMetadataFiller> DoCreateMetadataFiller() const override {
83+
return std::make_shared<TMetadataFromTable>();
84+
}
85+
public:
86+
static const inline TFactory::TRegistrator<TTableSysViewPolicy> Registrator = TFactory::TRegistrator<TTableSysViewPolicy>(TString(::NKikimr::NSysView::TablePrimaryIndexOptimizerStatsName));
87+
88+
};
89+
90+
}

‎ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/ya.make‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
LIBRARY()
2+
3+
PEERDIR(
4+
ydb/core/tx/columnshard/engines/reader/sys_view/abstract
5+
)
6+
7+
SRCS(
8+
GLOBAL optimizer.cpp
9+
)
10+
11+
END()
12+

‎ydb/core/tx/columnshard/engines/reader/sys_view/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ PEERDIR(
66
ydb/core/tx/columnshard/engines/reader/sys_view/chunks
77
ydb/core/tx/columnshard/engines/reader/sys_view/constructor
88
ydb/core/tx/columnshard/engines/reader/sys_view/granules
9+
ydb/core/tx/columnshard/engines/reader/sys_view/optimizer
910
)
1011

1112
SRCS(

‎ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,26 @@ class TOptimizationPriority {
5151

5252
};
5353

54+
class TTaskDescription {
55+
private:
56+
YDB_READONLY(ui64, TaskId, 0);
57+
YDB_ACCESSOR_DEF(TString, Start);
58+
YDB_ACCESSOR_DEF(TString, Finish);
59+
YDB_ACCESSOR_DEF(TString, Details);
60+
YDB_ACCESSOR_DEF(ui64, WeightCategory);
61+
YDB_ACCESSOR_DEF(i64, Weight);
62+
public:
63+
TTaskDescription(const ui64 taskId)
64+
: TaskId(taskId)
65+
{
66+
67+
}
68+
69+
bool operator<(const TTaskDescription& item) const {
70+
return TaskId < item.TaskId;
71+
}
72+
};
73+
5474
class IOptimizerPlanner {
5575
private:
5676
const ui64 PathId;
@@ -67,6 +87,7 @@ class IOptimizerPlanner {
6787
return NJson::JSON_NULL;
6888
}
6989
virtual bool DoIsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const = 0;
90+
virtual std::vector<TTaskDescription> DoGetTasksDescription() const = 0;
7091

7192
public:
7293
using TFactory = NObjectFactory::TObjectFactory<IOptimizerPlanner, TString>;
@@ -76,6 +97,10 @@ class IOptimizerPlanner {
7697

7798
}
7899

100+
std::vector<TTaskDescription> GetTasksDescription() const {
101+
return DoGetTasksDescription();
102+
}
103+
79104
class TModificationGuard: TNonCopyable {
80105
private:
81106
IOptimizerPlanner& Owner;

0 commit comments

Comments
 (0)