Skip to content

Commit 11f0b12

Browse files
swalrus1swalrus1
authored
Merge b4b11c1 into 250822e 
2 parents 250822e + b4b11c1 commit 11f0b12

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+967
-392
lines changed

‎ydb/core/protos/counters_columnshard.proto

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ enum ECumulativeCounters {
6565
COUNTER_PLAN_STEP_ACCEPTED = 9 [(CounterOpts) = {Name: "PlanStepAccepted"}];
6666
COUNTER_SCANNED_ROWS = 10 [(CounterOpts) = {Name: "ScannedRows"}];
6767
COUNTER_SCANNED_BYTES = 11 [(CounterOpts) = {Name: "ScannedBytes"}];
68-
COUNTER_UPSERT_BLOBS_WRITTEN = 12 [(CounterOpts) = {Name: "UpsertBlobsWritten"}];
69-
COUNTER_UPSERT_BYTES_WRITTEN = 13 [(CounterOpts) = {Name: "UpsertBytesWritten"}];
68+
COUNTER_OPERATIONS_BLOBS_WRITTEN = 12 [(CounterOpts) = {Name: "OperationsBlobsWritten"}];
69+
COUNTER_OPERATIONS_BYTES_WRITTEN = 13 [(CounterOpts) = {Name: "OperationsBytesWritten"}];
7070
COUNTER_INDEXING_BLOBS_WRITTEN = 14 [(CounterOpts) = {Name: "IndexingBlobsWritten"}];
7171
COUNTER_INDEXING_BYTES_WRITTEN = 15 [(CounterOpts) = {Name: "IndexingBytesWritten"}];
7272
COUNTER_COMPACTION_BLOBS_WRITTEN = 16 [(CounterOpts) = {Name: "CompactionBlobsWritten"}];
@@ -135,6 +135,10 @@ enum ECumulativeCounters {
135135
COUNTER_READING_EXPORTED_BLOBS = 79 [(CounterOpts) = {Name: "ReadingExportedBlobs"}];
136136
COUNTER_READING_EXPORTED_BYTES = 80 [(CounterOpts) = {Name: "ReadingExportedBytes"}];
137137
COUNTER_READING_EXPORTED_RANGES = 81 [(CounterOpts) = {Name: "ReadingExportedRanges"}];
138+
COUNTER_PLANNED_TX_COMPLETED = 82 [(CounterOpts) = {Name: "PlannedTxCompleted"}];
139+
COUNTER_IMMEDIATE_TX_COMPLETED = 83 [(CounterOpts) = {Name: "ImmediateTxCompleted"}];
140+
COUNTER_ROWS_ERASED = 84 [(CounterOpts) = {Name: "RowsErased"}];
141+
COUNTER_OPERATIONS_ROWS_WRITTEN = 85 [(CounterOpts) = {Name: "OperationsRowsWritten"}];
138142
}
139143

140144
enum EPercentileCounters {

‎ydb/core/tablet/tablet_counters_aggregator.cpp

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -765,10 +765,16 @@ class TTabletMon {
765765
TCounterPtr DatashardSizeBytes;
766766
TCounterPtr DatashardCacheHitBytes;
767767
TCounterPtr DatashardCacheMissBytes;
768+
TCounterPtr ColumnShardReadRows_;
769+
TCounterPtr ColumnShardReadBytes_;
768770
TCounterPtr ColumnShardScanRows_;
769771
TCounterPtr ColumnShardScanBytes_;
772+
TCounterPtr ColumnShardWriteRows_;
773+
TCounterPtr ColumnShardWriteBytes_;
770774
TCounterPtr ColumnShardBulkUpsertRows_;
771775
TCounterPtr ColumnShardBulkUpsertBytes_;
776+
TCounterPtr ColumnShardEraseRows_;
777+
TCounterPtr ColumnShardEraseBytes_;
772778
TCounterPtr ResourcesStorageUsedBytes;
773779
TCounterPtr ResourcesStorageUsedBytesOnSsd;
774780
TCounterPtr ResourcesStorageUsedBytesOnHdd;
@@ -787,6 +793,7 @@ class TTabletMon {
787793
TCounterPtr ResourcesStreamReservedStorageLimit;
788794

789795
THistogramPtr ShardCpuUtilization;
796+
THistogramPtr ColumnShardCpuUtilization;
790797

791798
TCounterPtr RowUpdates;
792799
TCounterPtr RowUpdateBytes;
@@ -808,8 +815,11 @@ class TTabletMon {
808815

809816
TCounterPtr ColumnShardScannedBytes_;
810817
TCounterPtr ColumnShardScannedRows_;
811-
TCounterPtr ColumnShardUpsertBlobsWritten_;
812-
TCounterPtr ColumnShardUpsertBytesWritten_;
818+
TCounterPtr ColumnShardOperationsRowsWritten_;
819+
TCounterPtr ColumnShardOperationsBytesWritten_;
820+
TCounterPtr ColumnShardErasedBytes_;
821+
TCounterPtr ColumnShardErasedRows_;
822+
THistogramPtr ColumnShardConsumedCpuHistogram;
813823

814824
TCounterPtr DiskSpaceTablesTotalBytes;
815825
TCounterPtr DiskSpaceTablesTotalBytesOnSsd;
@@ -859,14 +869,26 @@ class TTabletMon {
859869
DatashardCacheMissBytes = ydbGroup->GetNamedCounter("name",
860870
"table.datashard.cache_miss.bytes", true);
861871

872+
ColumnShardReadRows_ = ydbGroup->GetNamedCounter("name",
873+
"table.columnshard.read.rows", true);
874+
ColumnShardReadBytes_ = ydbGroup->GetNamedCounter("name",
875+
"table.columnshard.read.bytes", true);
862876
ColumnShardScanRows_ = ydbGroup->GetNamedCounter("name",
863877
"table.columnshard.scan.rows", true);
864878
ColumnShardScanBytes_ = ydbGroup->GetNamedCounter("name",
865879
"table.columnshard.scan.bytes", true);
880+
ColumnShardWriteRows_ = ydbGroup->GetNamedCounter("name",
881+
"table.columnshard.write.rows", true);
882+
ColumnShardWriteBytes_ = ydbGroup->GetNamedCounter("name",
883+
"table.columnshard.write.bytes", true);
866884
ColumnShardBulkUpsertRows_ = ydbGroup->GetNamedCounter("name",
867885
"table.columnshard.bulk_upsert.rows", true);
868886
ColumnShardBulkUpsertBytes_ = ydbGroup->GetNamedCounter("name",
869887
"table.columnshard.bulk_upsert.bytes", true);
888+
ColumnShardEraseRows_ = ydbGroup->GetNamedCounter("name",
889+
"table.columnshard.erase.rows", true);
890+
ColumnShardEraseBytes_ = ydbGroup->GetNamedCounter("name",
891+
"table.columnshard.erase.bytes", true);
870892

871893
ResourcesStorageUsedBytes = ydbGroup->GetNamedCounter("name",
872894
"resources.storage.used_bytes", false);
@@ -908,6 +930,8 @@ class TTabletMon {
908930

909931
ShardCpuUtilization = ydbGroup->GetNamedHistogram("name",
910932
"table.datashard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false);
933+
ColumnShardCpuUtilization = ydbGroup->GetNamedHistogram("name",
934+
"table.columnshard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false);
911935
};
912936

913937
void Initialize(::NMonitoring::TDynamicCounterPtr counters, bool hasDatashard, bool hasSchemeshard, bool hasColumnShard) {
@@ -943,8 +967,11 @@ class TTabletMon {
943967

944968
ColumnShardScannedBytes_ = appGroup->GetCounter("ColumnShard/ScannedBytes");
945969
ColumnShardScannedRows_ = appGroup->GetCounter("ColumnShard/ScannedRows");
946-
ColumnShardUpsertBlobsWritten_ = appGroup->GetCounter("ColumnShard/UpsertBlobsWritten");
947-
ColumnShardUpsertBytesWritten_ = appGroup->GetCounter("ColumnShard/UpsertBytesWritten");
970+
ColumnShardOperationsRowsWritten_ = appGroup->GetCounter("ColumnShard/OperationsRowsWritten");
971+
ColumnShardOperationsBytesWritten_ = appGroup->GetCounter("ColumnShard/OperationsBytesWritten");
972+
ColumnShardErasedBytes_ = appGroup->GetCounter("ColumnShard/BytesErased");
973+
ColumnShardErasedRows_ = appGroup->GetCounter("ColumnShard/RowsErased");
974+
ColumnShardConsumedCpuHistogram = appGroup->FindHistogram("HIST(ConsumedCPU)");
948975
}
949976

950977
if (hasSchemeshard && !DiskSpaceTablesTotalBytes) {
@@ -990,10 +1017,20 @@ class TTabletMon {
9901017
}
9911018

9921019
if (ColumnShardScannedBytes_) {
1020+
ColumnShardReadRows_->Set(0);
1021+
ColumnShardReadBytes_->Set(0);
9931022
ColumnShardScanRows_->Set(ColumnShardScannedRows_->Val());
9941023
ColumnShardScanBytes_->Set(ColumnShardScannedBytes_->Val());
995-
ColumnShardBulkUpsertRows_->Set(ColumnShardUpsertBlobsWritten_->Val());
996-
ColumnShardBulkUpsertBytes_->Set(ColumnShardUpsertBytesWritten_->Val());
1024+
ColumnShardWriteRows_->Set(ColumnShardOperationsRowsWritten_->Val());
1025+
ColumnShardWriteBytes_->Set(ColumnShardOperationsBytesWritten_->Val());
1026+
ColumnShardBulkUpsertRows_->Set(ColumnShardOperationsRowsWritten_->Val());
1027+
ColumnShardBulkUpsertBytes_->Set(ColumnShardOperationsBytesWritten_->Val());
1028+
ColumnShardEraseRows_->Set(ColumnShardErasedRows_->Val());
1029+
ColumnShardEraseBytes_->Set(ColumnShardErasedBytes_->Val());
1030+
1031+
if (ColumnShardConsumedCpuHistogram) {
1032+
TransferBuckets(ColumnShardCpuUtilization, ColumnShardConsumedCpuHistogram);
1033+
}
9971034
}
9981035

9991036
if (DiskSpaceTablesTotalBytes) {

‎ydb/core/tx/columnshard/background_controller.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22
#include "engines/changes/abstract/compaction_info.h"
33
#include "engines/portions/meta.h"
4+
#include <ydb/core/tx/columnshard/counters/counters_manager.h>
45

56
namespace NKikimr::NOlap {
67
class TColumnEngineChanges;
@@ -15,11 +16,16 @@ class TBackgroundController {
1516
using TCurrentCompaction = THashMap<ui64, NOlap::TPlanCompactionInfo>;
1617
TCurrentCompaction ActiveCompactionInfo;
1718

19+
std::shared_ptr<TBackgroundControllerCounters> Counters;
1820
bool ActiveCleanupPortions = false;
1921
bool ActiveCleanupTables = false;
2022
bool ActiveCleanupInsertTable = false;
2123
YDB_READONLY(TMonotonic, LastIndexationInstant, TMonotonic::Zero());
2224
public:
25+
TBackgroundController(std::shared_ptr<TBackgroundControllerCounters> counters)
26+
: Counters(std::move(counters)) {
27+
}
28+
2329
THashSet<NOlap::TPortionAddress> GetConflictTTLPortions() const;
2430
THashSet<NOlap::TPortionAddress> GetConflictCompactionPortions() const;
2531

@@ -29,6 +35,7 @@ class TBackgroundController {
2935
bool StartCompaction(const NOlap::TPlanCompactionInfo& info);
3036
void FinishCompaction(const NOlap::TPlanCompactionInfo& info) {
3137
Y_ABORT_UNLESS(ActiveCompactionInfo.erase(info.GetPathId()));
38+
Counters->OnCompactionFinish(info.GetPathId());
3239
}
3340
const TCurrentCompaction& GetActiveCompaction() const {
3441
return ActiveCompactionInfo;

‎ydb/core/tx/columnshard/blobs_action/bs/write.cpp

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,10 @@ void TWriteAction::DoOnCompleteTxAfterWrite(NColumnShard::TColumnShard& self, co
1515
ui64 blobsWritten = BlobBatch.GetBlobCount();
1616
ui64 bytesWritten = BlobBatch.GetTotalSize();
1717
if (blobsWroteSuccessfully) {
18-
self.IncCounter(NColumnShard::COUNTER_UPSERT_BLOBS_WRITTEN, blobsWritten);
19-
self.IncCounter(NColumnShard::COUNTER_UPSERT_BYTES_WRITTEN, bytesWritten);
20-
// self.IncCounter(NColumnShard::COUNTER_RAW_BYTES_UPSERTED, insertedBytes);
21-
self.IncCounter(NColumnShard::COUNTER_WRITE_SUCCESS);
18+
self.Counters.GetTabletCounters()->OnWriteSuccess(blobsWritten, bytesWritten);
2219
Manager->SaveBlobBatchOnComplete(std::move(BlobBatch));
2320
} else {
24-
self.IncCounter(NColumnShard::COUNTER_WRITE_FAIL);
21+
self.Counters.GetTabletCounters()->OnWriteFailure();
2522
}
2623
}
2724

‎ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ bool TTxWrite::Execute(TTransactionContext& txc, const TActorContext&) {
7070

7171
if (!InsertOneBlob(txc, i, writeId)) {
7272
LOG_S_DEBUG(TxPrefix() << "duplicate writeId " << (ui64)writeId << TxSuffix());
73-
Self->IncCounter(COUNTER_WRITE_DUPLICATE);
73+
Self->Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_DUPLICATE);
7474
}
7575
}
7676
}
@@ -140,10 +140,10 @@ void TTxWrite::Complete(const TActorContext& ctx) {
140140
}
141141
for (ui32 i = 0; i < buffer.GetAggregations().size(); ++i) {
142142
const auto& writeMeta = buffer.GetAggregations()[i]->GetWriteMeta();
143-
Self->CSCounters.OnWriteTxComplete(now - writeMeta.GetWriteStartInstant());
144-
Self->CSCounters.OnSuccessWriteResponse();
143+
Self->Counters.GetCSCounters().OnWriteTxComplete(now - writeMeta.GetWriteStartInstant());
144+
Self->Counters.GetCSCounters().OnSuccessWriteResponse();
145145
}
146-
146+
Self->Counters.GetTabletCounters()->IncCounter(COUNTER_IMMEDIATE_TX_COMPLETED);
147147
}
148148

149149
}

0 commit comments

Comments
 (0)