Skip to content

Commit 9f24faf

Browse files
swalrus1swalrus1
authored
Merge 927727c into 4a1f83b 
2 parents 4a1f83b + 927727c commit 9f24faf

19 files changed

+181
-118
lines changed

‎ydb/core/tx/columnshard/columnshard.cpp

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -327,14 +327,9 @@ void TColumnShard::FillOlapStats(
327327
resourceMetrics->Fill(*ev->Record.MutableTabletMetrics());
328328
}
329329

330-
TTableStatsBuilder statsBuilder(*ev->Record.MutableTableStats());
331-
statsBuilder.FillColumnTableStats(*Counters.GetColumnTablesCounters());
332-
statsBuilder.FillTabletStats(*Counters.GetTabletCounters());
333-
statsBuilder.FillBackgroundControllerStats(*Counters.GetBackgroundControllerCounters());
334-
statsBuilder.FillScanCountersStats(Counters.GetScanCounters());
335-
statsBuilder.FillExecutorStats(*Executor());
336330
if (TablesManager.HasPrimaryIndex()) {
337-
statsBuilder.FillColumnEngineStats(TablesManager.MutablePrimaryIndex().GetTotalStats());
331+
TTableStatsBuilder statsBuilder(Counters, Executor(), TablesManager.MutablePrimaryIndex());
332+
statsBuilder.FillTotalTableStats(*ev->Record.MutableTableStats());
338333
}
339334
}
340335

@@ -343,6 +338,7 @@ void TColumnShard::FillColumnTableStats(
343338
std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev
344339
) {
345340
auto tables = TablesManager.GetTables();
341+
TTableStatsBuilder tableStatsBuilder(Counters, Executor(), TablesManager.MutablePrimaryIndex());
346342

347343
LOG_S_DEBUG("There are stats for " << tables.size() << " tables");
348344
for (const auto& [pathId, _] : tables) {
@@ -360,18 +356,7 @@ void TColumnShard::FillColumnTableStats(
360356
resourceMetrics->Fill(*periodicTableStats->MutableTabletMetrics());
361357
}
362358

363-
TTableStatsBuilder statsBuilder(*periodicTableStats->MutableTableStats());
364-
statsBuilder.FillColumnTableStats(*Counters.GetColumnTablesCounters()->GetPathIdCounter(pathId));
365-
statsBuilder.FillTabletStats(*Counters.GetTabletCounters());
366-
statsBuilder.FillBackgroundControllerStats(*Counters.GetBackgroundControllerCounters(), pathId);
367-
statsBuilder.FillScanCountersStats(Counters.GetScanCounters());
368-
statsBuilder.FillExecutorStats(*Executor());
369-
if (TablesManager.HasPrimaryIndex()) {
370-
auto columnEngineStats = TablesManager.GetPrimaryIndexSafe().GetStats().FindPtr(pathId);
371-
if (columnEngineStats && *columnEngineStats) {
372-
statsBuilder.FillColumnEngineStats(**columnEngineStats);
373-
}
374-
}
359+
tableStatsBuilder.FillTableStats(pathId, *(periodicTableStats->MutableTableStats()));
375360

376361
LOG_S_TRACE("Add stats for table, tableLocalID=" << pathId);
377362
}

‎ydb/core/tx/columnshard/columnshard__scan.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvScan::TPtr& ev, const TActorContext
3030
return;
3131
}
3232

33-
Counters.GetColumnTablesCounters()->GetPathIdCounter(record.GetLocalPathId())->OnAccess();
33+
Counters.GetColumnTablesCounters()->GetPathIdCounter(record.GetLocalPathId())->OnReadEvent();
3434
ScanTxInFlight.insert({txId, TAppData::TimeProvider->Now()});
3535
Counters.GetTabletCounters()->SetCounter(COUNTER_SCAN_IN_FLY, ScanTxInFlight.size());
3636
Execute(new NOlap::NReader::TTxScan(this, ev), ctx);

‎ydb/core/tx/columnshard/columnshard__write.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo
8484
auto baseAggregations = wBuffer.GetAggregations();
8585
wBuffer.InitReplyReceived(TMonotonic::Now());
8686

87-
auto wg = Counters.GetWritesMonitor()->OnFinishWrite(wBuffer.GetSumSize(), wBuffer.GetAggregations().size());
87+
Counters.GetWritesMonitor()->OnFinishWrite(wBuffer.GetSumSize(), wBuffer.GetAggregations().size());
8888

8989
for (auto&& aggr : baseAggregations) {
9090
const auto& writeMeta = aggr->GetWriteMeta();
@@ -156,7 +156,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
156156
const TString dedupId = record.GetDedupId();
157157
const auto source = ev->Sender;
158158

159-
Counters.GetColumnTablesCounters()->GetPathIdCounter(tableId)->OnUpdate();
159+
Counters.GetColumnTablesCounters()->GetPathIdCounter(tableId)->OnWriteEvent();
160160

161161
std::optional<ui32> granuleShardingVersion;
162162
if (record.HasGranuleShardingVersion()) {
@@ -375,7 +375,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor
375375
return;
376376
}
377377

378-
auto wg = Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize());
378+
Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize());
379379

380380
std::optional<ui32> granuleShardingVersionId;
381381
if (record.HasGranuleShardingVersionId()) {

‎ydb/core/tx/columnshard/counters/aggregation/table_stats.cpp

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎ydb/core/tx/columnshard/counters/aggregation/table_stats.h

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,49 @@
11
#pragma once
22

3-
#include <ydb/core/tx/columnshard/counters/scan.h>
3+
#include <ydb/core/protos/table_stats.pb.h>
44
#include <ydb/core/tablet_flat/tablet_flat_executor.h>
5-
#include <ydb/core/tx/columnshard/counters/column_tables.h>
6-
#include <ydb/core/tx/columnshard/counters/tablet_counters.h>
7-
#include <ydb/core/tx/columnshard/counters/background_controller.h>
5+
#include <ydb/core/tx/columnshard/counters/counters_manager.h>
86
#include <ydb/core/tx/columnshard/engines/column_engine.h>
9-
#include <ydb/core/protos/table_stats.pb.h>
107

118
namespace NKikimr::NColumnShard {
129

1310
class TTableStatsBuilder {
1411
private:
15-
::NKikimrTableStats::TTableStats& TableStats;
12+
TCountersManager& Counters;
13+
const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor& Executor;
14+
NOlap::IColumnEngine& ColumnEngine;
1615

1716
public:
18-
TTableStatsBuilder(::NKikimrTableStats::TTableStats& tableStats)
19-
: TableStats(tableStats) {
17+
TTableStatsBuilder(
18+
TCountersManager& counters, const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor* executor, NOlap::IColumnEngine& columnEngine)
19+
: Counters(counters)
20+
, Executor(*executor)
21+
, ColumnEngine(columnEngine) {
2022
}
2123

22-
void FillColumnTableStats(const TSingleColumnTableCounters& stats);
23-
void FillColumnTableStats(const TColumnTablesCounters& stats);
24-
25-
void FillTabletStats(const TTabletCountersHandle& stats);
24+
void FillTableStats(ui64 pathId, ::NKikimrTableStats::TTableStats& tableStats) {
25+
Counters.FillTableStats(pathId, tableStats);
2626

27-
void FillBackgroundControllerStats(const TBackgroundControllerCounters& stats, ui64 pathId);
28-
void FillBackgroundControllerStats(const TBackgroundControllerCounters& stats);
27+
auto columnEngineStats = ColumnEngine.GetStats().FindPtr(pathId);
28+
if (columnEngineStats && *columnEngineStats) {
29+
auto activeStats = (*columnEngineStats)->Active();
30+
tableStats.SetRowCount(activeStats.Rows);
31+
tableStats.SetDataSize(activeStats.Bytes);
32+
tableStats.SetPartCount(activeStats.Portions);
33+
}
34+
}
2935

30-
void FillScanCountersStats(const TScanCounters& stats);
36+
void FillTotalTableStats(::NKikimrTableStats::TTableStats& tableStats) {
37+
Counters.FillTotalTableStats(tableStats);
3138

32-
void FillExecutorStats(const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor& executor);
39+
tableStats.SetInFlightTxCount(Executor.GetStats().TxInFly);
40+
tableStats.SetHasLoanedParts(Executor.HasLoanedParts());
3341

34-
void FillColumnEngineStats(const NOlap::TColumnEngineStats& stats);
42+
auto activeStats = ColumnEngine.GetTotalStats().Active();
43+
tableStats.SetRowCount(activeStats.Rows);
44+
tableStats.SetDataSize(activeStats.Bytes);
45+
tableStats.SetPartCount(activeStats.Portions);
46+
}
3547
};
3648

3749
} // namespace NKikimr::NColumnShard

‎ydb/core/tx/columnshard/counters/aggregation/ya.make

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
LIBRARY()
22

3-
SRCS(
4-
table_stats.cpp
5-
)
3+
SRCS()
64

75
PEERDIR(
86
ydb/core/protos

‎ydb/core/tx/columnshard/counters/column_tables.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ class TSingleColumnTableCounters {
4747
, TotalLastUpdateTime(owner.LastUpdateTime) {
4848
}
4949

50-
void OnAccess() {
50+
void OnReadEvent() {
5151
UpdateLastAccessTime(TAppData::TimeProvider->Now());
5252
}
5353

54-
void OnUpdate() {
54+
void OnWriteEvent() {
5555
TInstant now = TAppData::TimeProvider->Now();
5656
UpdateLastUpdateTime(now);
5757
UpdateLastAccessTime(now);

‎ydb/core/tx/columnshard/counters/columnshard.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ namespace NKikimr::NColumnShard {
99
TCSCounters::TCSCounters(std::shared_ptr<const TTabletCountersHandle> tabletCounters)
1010
: TBase("CS")
1111
, TabletCounters(std::move(tabletCounters))
12-
, Initialization(*this) {
12+
, Initialization(*this)
13+
, TxProgress(*this) {
1314
Y_ABORT_UNLESS(TabletCounters);
1415

1516
StartBackgroundCount = TBase::GetDeriviative("StartBackground/Count");

‎ydb/core/tx/columnshard/counters/columnshard.h

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
#pragma once
22
#include "common/owner.h"
33

4-
#include <library/cpp/monlib/dynamic_counters/counters.h>
4+
#include <ydb/core/tx/columnshard/counters/tablet_counters.h>
55

6+
#include <library/cpp/monlib/dynamic_counters/counters.h>
67
#include <util/generic/hash_set.h>
7-
#include <ydb/core/tx/columnshard/counters/tablet_counters.h>
88

99
namespace NKikimr::NColumnShard {
1010

@@ -30,7 +30,6 @@ class TCSInitialization: public TCommonCountersOwner {
3030
const NMonitoring::THistogramPtr HistogramSwitchToWorkFromCreateDurationMs;
3131

3232
public:
33-
3433
void OnTxInitFinished(const TDuration d) const {
3534
HistogramTxInitDurationMs->Collect(d.MilliSeconds());
3635
}
@@ -66,6 +65,83 @@ class TCSInitialization: public TCommonCountersOwner {
6665
}
6766
};
6867

68+
class TTxProgressCounters: public TCommonCountersOwner {
69+
private:
70+
using TBase = TCommonCountersOwner;
71+
using TOpType = TString;
72+
73+
class TProgressCounters: public TCommonCountersOwner {
74+
private:
75+
using TBase = TCommonCountersOwner;
76+
77+
public:
78+
NMonitoring::TDynamicCounters::TCounterPtr RegisterTx;
79+
NMonitoring::TDynamicCounters::TCounterPtr RegisterTxWithDeadline;
80+
NMonitoring::TDynamicCounters::TCounterPtr StartProposeOnExecute;
81+
NMonitoring::TDynamicCounters::TCounterPtr StartProposeOnComplete;
82+
NMonitoring::TDynamicCounters::TCounterPtr FinishProposeOnExecute;
83+
NMonitoring::TDynamicCounters::TCounterPtr FinishProposeOnComplete;
84+
NMonitoring::TDynamicCounters::TCounterPtr FinishPlannedTx;
85+
NMonitoring::TDynamicCounters::TCounterPtr AbortTx;
86+
87+
TProgressCounters(TBase& owner)
88+
: TBase(owner)
89+
, RegisterTx(owner.GetDeriviative("RegisterTx"))
90+
, RegisterTxWithDeadline(owner.GetDeriviative("RegisterTxWithDeadline"))
91+
, StartProposeOnExecute(owner.GetDeriviative("StartProposeOnExecute"))
92+
, StartProposeOnComplete(owner.GetDeriviative("StartProposeOnComplete"))
93+
, FinishProposeOnExecute(owner.GetDeriviative("FinishProposeOnExecute"))
94+
, FinishProposeOnComplete(owner.GetDeriviative("FinishProposeOnComplete"))
95+
, FinishPlannedTx(owner.GetDeriviative("FinishPlannedTx"))
96+
, AbortTx(owner.GetDeriviative("AbortTx")) {
97+
}
98+
};
99+
100+
THashMap<TOpType, TProgressCounters> SubGroups;
101+
102+
public:
103+
void OnRegisterTx(const TOpType& opType) {
104+
GetSubGroup(opType).RegisterTx->Add(1);
105+
}
106+
107+
void OnRegisterTxWithDeadline(const TOpType& opType) {
108+
GetSubGroup(opType).RegisterTxWithDeadline->Add(1);
109+
}
110+
111+
void OnStartProposeOnExecute(const TOpType& opType) {
112+
GetSubGroup(opType).StartProposeOnExecute->Add(1);
113+
}
114+
115+
void OnStartProposeOnComplete(const TOpType& opType) {
116+
GetSubGroup(opType).StartProposeOnComplete->Add(1);
117+
}
118+
119+
void OnFinishProposeOnExecute(const TOpType& opType) {
120+
GetSubGroup(opType).FinishProposeOnExecute->Add(1);
121+
}
122+
123+
void OnFinishProposeOnComplete(const TOpType& opType) {
124+
GetSubGroup(opType).FinishProposeOnComplete->Add(1);
125+
}
126+
127+
void OnFinishPlannedTx(const TOpType& opType) {
128+
GetSubGroup(opType).FinishPlannedTx->Add(1);
129+
}
130+
131+
void OnAbortTx(const TOpType& opType) {
132+
GetSubGroup(opType).AbortTx->Add(1);
133+
}
134+
135+
TTxProgressCounters(TCommonCountersOwner& owner)
136+
: TBase(owner, "TxProgress") {
137+
}
138+
139+
private:
140+
TProgressCounters& GetSubGroup(const TOpType& opType) {
141+
return SubGroups.try_emplace(opType, *this).first->second;
142+
}
143+
};
144+
69145
class TCSCounters: public TCommonCountersOwner {
70146
private:
71147
using TBase = TCommonCountersOwner;
@@ -122,6 +198,7 @@ class TCSCounters: public TCommonCountersOwner {
122198

123199
public:
124200
const TCSInitialization Initialization;
201+
TTxProgressCounters TxProgress;
125202

126203
void OnStartWriteRequest() const {
127204
WriteRequests->Add(1);

‎ydb/core/tx/columnshard/counters/counters_manager.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,18 @@ class TCountersManager {
4242
, ColumnTablesCounters(std::make_shared<TColumnTablesCounters>())
4343
, SubscribeCounters(std::make_shared<NOlap::NResourceBroker::NSubscribe::TSubscriberCounters>()) {
4444
}
45+
46+
void FillTableStats(ui64 pathId, ::NKikimrTableStats::TTableStats& tableStats) {
47+
ColumnTablesCounters->GetPathIdCounter(pathId)->FillStats(tableStats);
48+
BackgroundControllerCounters->FillStats(pathId, tableStats);
49+
}
50+
51+
void FillTotalTableStats(::NKikimrTableStats::TTableStats& tableStats) {
52+
ColumnTablesCounters->FillStats(tableStats);
53+
TabletCounters->FillStats(tableStats);
54+
BackgroundControllerCounters->FillTotalStats(tableStats);
55+
ScanCounters.FillStats(tableStats);
56+
}
4557
};
4658

4759
} // namespace NKikimr::NColumnShard

0 commit comments

Comments
 (0)