Skip to content

Commit 3bb8608

Browse files
swalrus1swalrus1
authored
Merge 3b066ed into e9a7b28 
2 parents e9a7b28 + 3b066ed commit 3bb8608

23 files changed

+316
-196
lines changed

‎ydb/core/tx/columnshard/columnshard.cpp

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -327,14 +327,9 @@ void TColumnShard::FillOlapStats(
327327
resourceMetrics->Fill(*ev->Record.MutableTabletMetrics());
328328
}
329329

330-
TTableStatsBuilder statsBuilder(*ev->Record.MutableTableStats());
331-
statsBuilder.FillColumnTableStats(*Counters.GetColumnTablesCounters());
332-
statsBuilder.FillTabletStats(*Counters.GetTabletCounters());
333-
statsBuilder.FillBackgroundControllerStats(*Counters.GetBackgroundControllerCounters());
334-
statsBuilder.FillScanCountersStats(Counters.GetScanCounters());
335-
statsBuilder.FillExecutorStats(*Executor());
336330
if (TablesManager.HasPrimaryIndex()) {
337-
statsBuilder.FillColumnEngineStats(TablesManager.MutablePrimaryIndex().GetTotalStats());
331+
TTableStatsBuilder statsBuilder(Counters, Executor(), TablesManager.MutablePrimaryIndex());
332+
statsBuilder.FillTotalTableStats(*ev->Record.MutableTableStats());
338333
}
339334
}
340335

@@ -343,6 +338,9 @@ void TColumnShard::FillColumnTableStats(
343338
std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev
344339
) {
345340
auto tables = TablesManager.GetTables();
341+
std::optional<TTableStatsBuilder> tableStatsBuilder =
342+
TablesManager.HasPrimaryIndex() ? std::make_optional<TTableStatsBuilder>(Counters, Executor(), TablesManager.MutablePrimaryIndex())
343+
: std::nullopt;
346344

347345
LOG_S_DEBUG("There are stats for " << tables.size() << " tables");
348346
for (const auto& [pathId, _] : tables) {
@@ -360,17 +358,8 @@ void TColumnShard::FillColumnTableStats(
360358
resourceMetrics->Fill(*periodicTableStats->MutableTabletMetrics());
361359
}
362360

363-
TTableStatsBuilder statsBuilder(*periodicTableStats->MutableTableStats());
364-
statsBuilder.FillColumnTableStats(*Counters.GetColumnTablesCounters()->GetPathIdCounter(pathId));
365-
statsBuilder.FillTabletStats(*Counters.GetTabletCounters());
366-
statsBuilder.FillBackgroundControllerStats(*Counters.GetBackgroundControllerCounters(), pathId);
367-
statsBuilder.FillScanCountersStats(Counters.GetScanCounters());
368-
statsBuilder.FillExecutorStats(*Executor());
369-
if (TablesManager.HasPrimaryIndex()) {
370-
auto columnEngineStats = TablesManager.GetPrimaryIndexSafe().GetStats().FindPtr(pathId);
371-
if (columnEngineStats && *columnEngineStats) {
372-
statsBuilder.FillColumnEngineStats(**columnEngineStats);
373-
}
361+
if (tableStatsBuilder) {
362+
tableStatsBuilder->FillTableStats(pathId, *(periodicTableStats->MutableTableStats()));
374363
}
375364

376365
LOG_S_TRACE("Add stats for table, tableLocalID=" << pathId);

‎ydb/core/tx/columnshard/columnshard__scan.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvScan::TPtr& ev, const TActorContext
3030
return;
3131
}
3232

33-
Counters.GetColumnTablesCounters()->GetPathIdCounter(record.GetLocalPathId())->OnAccess();
33+
Counters.GetColumnTablesCounters()->GetPathIdCounter(record.GetLocalPathId())->OnReadEvent();
3434
ScanTxInFlight.insert({txId, TAppData::TimeProvider->Now()});
3535
Counters.GetTabletCounters()->SetCounter(COUNTER_SCAN_IN_FLY, ScanTxInFlight.size());
3636
Execute(new NOlap::NReader::TTxScan(this, ev), ctx);

‎ydb/core/tx/columnshard/columnshard__write.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,22 @@ void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const
1616
Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL);
1717
switch (overloadReason) {
1818
case EOverloadStatus::Disk:
19-
Counters.GetCSCounters().OnWriteOverloadDisk();
19+
Counters.OnWriteOverloadDisk();
2020
break;
2121
case EOverloadStatus::InsertTable:
22-
Counters.GetCSCounters().OnWriteOverloadInsertTable(writeData.GetSize());
22+
Counters.OnWriteOverloadInsertTable(writeData.GetSize());
2323
break;
2424
case EOverloadStatus::OverloadMetadata:
25-
Counters.GetCSCounters().OnWriteOverloadMetadata(writeData.GetSize());
25+
Counters.OnWriteOverloadMetadata(writeData.GetSize());
2626
break;
2727
case EOverloadStatus::ShardTxInFly:
28-
Counters.GetCSCounters().OnWriteOverloadShardTx(writeData.GetSize());
28+
Counters.OnWriteOverloadShardTx(writeData.GetSize());
2929
break;
3030
case EOverloadStatus::ShardWritesInFly:
31-
Counters.GetCSCounters().OnWriteOverloadShardWrites(writeData.GetSize());
31+
Counters.OnWriteOverloadShardWrites(writeData.GetSize());
3232
break;
3333
case EOverloadStatus::ShardWritesSizeInFly:
34-
Counters.GetCSCounters().OnWriteOverloadShardWritesSize(writeData.GetSize());
34+
Counters.OnWriteOverloadShardWritesSize(writeData.GetSize());
3535
break;
3636
case EOverloadStatus::None:
3737
Y_ABORT("invalid function usage");
@@ -84,7 +84,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo
8484
auto baseAggregations = wBuffer.GetAggregations();
8585
wBuffer.InitReplyReceived(TMonotonic::Now());
8686

87-
auto wg = Counters.GetWritesMonitor()->OnFinishWrite(wBuffer.GetSumSize(), wBuffer.GetAggregations().size());
87+
Counters.GetWritesMonitor()->OnFinishWrite(wBuffer.GetSumSize(), wBuffer.GetAggregations().size());
8888

8989
for (auto&& aggr : baseAggregations) {
9090
const auto& writeMeta = aggr->GetWriteMeta();
@@ -156,7 +156,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
156156
const TString dedupId = record.GetDedupId();
157157
const auto source = ev->Sender;
158158

159-
Counters.GetColumnTablesCounters()->GetPathIdCounter(tableId)->OnUpdate();
159+
Counters.GetColumnTablesCounters()->GetPathIdCounter(tableId)->OnWriteEvent();
160160

161161
std::optional<ui32> granuleShardingVersion;
162162
if (record.HasGranuleShardingVersion()) {
@@ -375,7 +375,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor
375375
return;
376376
}
377377

378-
auto wg = Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize());
378+
Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize());
379379

380380
std::optional<ui32> granuleShardingVersionId;
381381
if (record.HasGranuleShardingVersionId()) {

‎ydb/core/tx/columnshard/columnshard_impl.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,15 @@ NTabletPipe::TClientConfig GetPipeClientConfig() {
6565
TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet)
6666
: TActor(&TThis::StateInit)
6767
, TTabletExecutedFlat(info, tablet, nullptr)
68+
, TabletCountersHolder(new TProtobufTabletCounters<ESimpleCounters_descriptor, ECumulativeCounters_descriptor,
69+
EPercentileCounters_descriptor, ETxTypes_descriptor>())
70+
, Counters(*TabletCountersHolder)
6871
, ProgressTxController(std::make_unique<TTxController>(*this))
6972
, StoragesManager(std::make_shared<NOlap::TStoragesManager>(*this))
7073
, DataLocksManager(std::make_shared<NOlap::NDataLocks::TManager>())
7174
, PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod(
7275
TSettings::DefaultPeriodicWakeupActivationPeriod))
7376
, StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval(TSettings::DefaultStatsReportInterval))
74-
, TabletCountersHolder(new TProtobufTabletCounters<ESimpleCounters_descriptor, ECumulativeCounters_descriptor,
75-
EPercentileCounters_descriptor, ETxTypes_descriptor>())
76-
, Counters(*TabletCountersHolder)
7777
, InFlightReadsTracker(StoragesManager)
7878
, TablesManager(StoragesManager, info->TabletID)
7979
, Subscribers(std::make_shared<NSubscriber::TManager>(*this))

‎ydb/core/tx/columnshard/columnshard_impl.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,9 @@ class TColumnShard
395395
}
396396

397397
private:
398+
std::unique_ptr<TTabletCountersBase> TabletCountersHolder;
399+
TCountersManager Counters;
400+
398401
std::unique_ptr<TTxController> ProgressTxController;
399402
std::unique_ptr<TOperationsManager> OperationsManager;
400403
std::shared_ptr<NOlap::NDataSharing::TSessionsManager> SharingSessionsManager;
@@ -441,9 +444,6 @@ class TColumnShard
441444
TActorId BufferizationWriteActorId;
442445
TActorId StatsReportPipe;
443446

444-
std::unique_ptr<TTabletCountersBase> TabletCountersHolder;
445-
TCountersManager Counters;
446-
447447
TInFlightReadsTracker InFlightReadsTracker;
448448
TTablesManager TablesManager;
449449
std::shared_ptr<NSubscriber::TManager> Subscribers;

‎ydb/core/tx/columnshard/counters/aggregation/table_stats.cpp

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎ydb/core/tx/columnshard/counters/aggregation/table_stats.h

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,49 @@
11
#pragma once
22

3-
#include <ydb/core/tx/columnshard/counters/scan.h>
3+
#include <ydb/core/protos/table_stats.pb.h>
44
#include <ydb/core/tablet_flat/tablet_flat_executor.h>
5-
#include <ydb/core/tx/columnshard/counters/column_tables.h>
6-
#include <ydb/core/tx/columnshard/counters/tablet_counters.h>
7-
#include <ydb/core/tx/columnshard/counters/background_controller.h>
5+
#include <ydb/core/tx/columnshard/counters/counters_manager.h>
86
#include <ydb/core/tx/columnshard/engines/column_engine.h>
9-
#include <ydb/core/protos/table_stats.pb.h>
107

118
namespace NKikimr::NColumnShard {
129

1310
class TTableStatsBuilder {
1411
private:
15-
::NKikimrTableStats::TTableStats& TableStats;
12+
TCountersManager& Counters;
13+
const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor& Executor;
14+
NOlap::IColumnEngine& ColumnEngine;
1615

1716
public:
18-
TTableStatsBuilder(::NKikimrTableStats::TTableStats& tableStats)
19-
: TableStats(tableStats) {
17+
TTableStatsBuilder(
18+
TCountersManager& counters, const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor* executor, NOlap::IColumnEngine& columnEngine)
19+
: Counters(counters)
20+
, Executor(*executor)
21+
, ColumnEngine(columnEngine) {
2022
}
2123

22-
void FillColumnTableStats(const TSingleColumnTableCounters& stats);
23-
void FillColumnTableStats(const TColumnTablesCounters& stats);
24-
25-
void FillTabletStats(const TTabletCountersHandle& stats);
24+
void FillTableStats(ui64 pathId, ::NKikimrTableStats::TTableStats& tableStats) {
25+
Counters.FillTableStats(pathId, tableStats);
2626

27-
void FillBackgroundControllerStats(const TBackgroundControllerCounters& stats, ui64 pathId);
28-
void FillBackgroundControllerStats(const TBackgroundControllerCounters& stats);
27+
auto columnEngineStats = ColumnEngine.GetStats().FindPtr(pathId);
28+
if (columnEngineStats && *columnEngineStats) {
29+
auto activeStats = (*columnEngineStats)->Active();
30+
tableStats.SetRowCount(activeStats.Rows);
31+
tableStats.SetDataSize(activeStats.Bytes);
32+
tableStats.SetPartCount(activeStats.Portions);
33+
}
34+
}
2935

30-
void FillScanCountersStats(const TScanCounters& stats);
36+
void FillTotalTableStats(::NKikimrTableStats::TTableStats& tableStats) {
37+
Counters.FillTotalTableStats(tableStats);
3138

32-
void FillExecutorStats(const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor& executor);
39+
tableStats.SetInFlightTxCount(Executor.GetStats().TxInFly);
40+
tableStats.SetHasLoanedParts(Executor.HasLoanedParts());
3341

34-
void FillColumnEngineStats(const NOlap::TColumnEngineStats& stats);
42+
auto activeStats = ColumnEngine.GetTotalStats().Active();
43+
tableStats.SetRowCount(activeStats.Rows);
44+
tableStats.SetDataSize(activeStats.Bytes);
45+
tableStats.SetPartCount(activeStats.Portions);
46+
}
3547
};
3648

37-
} // namespace NKikimr::NColumnShard
49+
} // namespace NKikimr::NColumnShard

‎ydb/core/tx/columnshard/counters/aggregation/ya.make

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
LIBRARY()
22

3-
SRCS(
4-
table_stats.cpp
5-
)
3+
SRCS()
64

75
PEERDIR(
86
ydb/core/protos

‎ydb/core/tx/columnshard/counters/column_tables.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ class TSingleColumnTableCounters {
4747
, TotalLastUpdateTime(owner.LastUpdateTime) {
4848
}
4949

50-
void OnAccess() {
50+
void OnReadEvent() {
5151
UpdateLastAccessTime(TAppData::TimeProvider->Now());
5252
}
5353

54-
void OnUpdate() {
54+
void OnWriteEvent() {
5555
TInstant now = TAppData::TimeProvider->Now();
5656
UpdateLastUpdateTime(now);
5757
UpdateLastAccessTime(now);

‎ydb/core/tx/columnshard/counters/columnshard.cpp

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,10 @@
66

77
namespace NKikimr::NColumnShard {
88

9-
TCSCounters::TCSCounters(std::shared_ptr<const TTabletCountersHandle> tabletCounters)
9+
TCSCounters::TCSCounters()
1010
: TBase("CS")
11-
, TabletCounters(std::move(tabletCounters))
12-
, Initialization(*this) {
13-
Y_ABORT_UNLESS(TabletCounters);
14-
11+
, Initialization(*this)
12+
, TxProgress(*this) {
1513
StartBackgroundCount = TBase::GetDeriviative("StartBackground/Count");
1614
TooEarlyBackgroundCount = TBase::GetDeriviative("TooEarlyBackground/Count");
1715
SetupCompactionCount = TBase::GetDeriviative("SetupCompaction/Count");

0 commit comments

Comments
 (0)