Update stable-24-3: add OLAP metrics to describe table

ydb/core/protos/counters_columnshard.proto

@@ -65,8 +65,8 @@ enum ECumulativeCounters {
     COUNTER_PLAN_STEP_ACCEPTED = 9                          [(CounterOpts) = {Name: "PlanStepAccepted"}];
     COUNTER_SCANNED_ROWS = 10                               [(CounterOpts) = {Name: "ScannedRows"}];
     COUNTER_SCANNED_BYTES = 11                              [(CounterOpts) = {Name: "ScannedBytes"}];
-    COUNTER_UPSERT_BLOBS_WRITTEN = 12                       [(CounterOpts) = {Name: "UpsertBlobsWritten"}];
-    COUNTER_UPSERT_BYTES_WRITTEN = 13                       [(CounterOpts) = {Name: "UpsertBytesWritten"}];
+    COUNTER_OPERATIONS_BLOBS_WRITTEN = 12                   [(CounterOpts) = {Name: "OperationsBlobsWritten"}];
+    COUNTER_OPERATIONS_BYTES_WRITTEN = 13                   [(CounterOpts) = {Name: "OperationsBytesWritten"}];
     COUNTER_INDEXING_BLOBS_WRITTEN = 14                     [(CounterOpts) = {Name: "IndexingBlobsWritten"}];
     COUNTER_INDEXING_BYTES_WRITTEN = 15                     [(CounterOpts) = {Name: "IndexingBytesWritten"}];
     COUNTER_COMPACTION_BLOBS_WRITTEN = 16                   [(CounterOpts) = {Name: "CompactionBlobsWritten"}];
@@ -135,6 +135,10 @@ enum ECumulativeCounters {
     COUNTER_READING_EXPORTED_BLOBS = 79                     [(CounterOpts) = {Name: "ReadingExportedBlobs"}];
     COUNTER_READING_EXPORTED_BYTES = 80                     [(CounterOpts) = {Name: "ReadingExportedBytes"}];
     COUNTER_READING_EXPORTED_RANGES = 81                    [(CounterOpts) = {Name: "ReadingExportedRanges"}];
+    COUNTER_PLANNED_TX_COMPLETED = 82                       [(CounterOpts) = {Name: "PlannedTxCompleted"}];
+    COUNTER_IMMEDIATE_TX_COMPLETED = 83                     [(CounterOpts) = {Name: "ImmediateTxCompleted"}];
+    COUNTER_ROWS_ERASED = 84                                [(CounterOpts) = {Name: "RowsErased"}];
+    COUNTER_OPERATIONS_ROWS_WRITTEN = 85                    [(CounterOpts) = {Name: "OperationsRowsWritten"}];
 }
 
 enum EPercentileCounters {

ydb/core/tablet/tablet_counters_aggregator.cpp

@@ -765,10 +765,16 @@ class TTabletMon {
         TCounterPtr DatashardSizeBytes;
         TCounterPtr DatashardCacheHitBytes;
         TCounterPtr DatashardCacheMissBytes;
+        TCounterPtr ColumnShardReadRows_;
+        TCounterPtr ColumnShardReadBytes_;
         TCounterPtr ColumnShardScanRows_;
         TCounterPtr ColumnShardScanBytes_;
+        TCounterPtr ColumnShardWriteRows_;
+        TCounterPtr ColumnShardWriteBytes_;
         TCounterPtr ColumnShardBulkUpsertRows_;
         TCounterPtr ColumnShardBulkUpsertBytes_;
+        TCounterPtr ColumnShardEraseRows_;
+        TCounterPtr ColumnShardEraseBytes_;
         TCounterPtr ResourcesStorageUsedBytes;
         TCounterPtr ResourcesStorageUsedBytesOnSsd;
         TCounterPtr ResourcesStorageUsedBytesOnHdd;
@@ -787,6 +793,7 @@ class TTabletMon {
         TCounterPtr ResourcesStreamReservedStorageLimit;
 
         THistogramPtr ShardCpuUtilization;
+        THistogramPtr ColumnShardCpuUtilization;
 
         TCounterPtr RowUpdates;
         TCounterPtr RowUpdateBytes;
@@ -808,8 +815,11 @@ class TTabletMon {
 
         TCounterPtr ColumnShardScannedBytes_;
         TCounterPtr ColumnShardScannedRows_;
-        TCounterPtr ColumnShardUpsertBlobsWritten_;
-        TCounterPtr ColumnShardUpsertBytesWritten_;
+        TCounterPtr ColumnShardOperationsRowsWritten_;
+        TCounterPtr ColumnShardOperationsBytesWritten_;
+        TCounterPtr ColumnShardErasedBytes_;
+        TCounterPtr ColumnShardErasedRows_;
+        THistogramPtr ColumnShardConsumedCpuHistogram;
 
         TCounterPtr DiskSpaceTablesTotalBytes;
         TCounterPtr DiskSpaceTablesTotalBytesOnSsd;
@@ -859,14 +869,26 @@ class TTabletMon {
             DatashardCacheMissBytes = ydbGroup->GetNamedCounter("name",
                 "table.datashard.cache_miss.bytes", true);
 
+            ColumnShardReadRows_ = ydbGroup->GetNamedCounter("name",
+                "table.columnshard.read.rows", true);
+            ColumnShardReadBytes_ = ydbGroup->GetNamedCounter("name",
+                "table.columnshard.read.bytes", true);
             ColumnShardScanRows_ = ydbGroup->GetNamedCounter("name",
                 "table.columnshard.scan.rows", true);
             ColumnShardScanBytes_ = ydbGroup->GetNamedCounter("name",
                 "table.columnshard.scan.bytes", true);
+            ColumnShardWriteRows_ = ydbGroup->GetNamedCounter("name",
+                "table.columnshard.write.rows", true);
+            ColumnShardWriteBytes_ = ydbGroup->GetNamedCounter("name",
+                "table.columnshard.write.bytes", true);
             ColumnShardBulkUpsertRows_ = ydbGroup->GetNamedCounter("name",
                 "table.columnshard.bulk_upsert.rows", true);
             ColumnShardBulkUpsertBytes_ = ydbGroup->GetNamedCounter("name",
                 "table.columnshard.bulk_upsert.bytes", true);
+            ColumnShardEraseRows_ = ydbGroup->GetNamedCounter("name",
+                "table.columnshard.erase.rows", true);
+            ColumnShardEraseBytes_ = ydbGroup->GetNamedCounter("name",
+                "table.columnshard.erase.bytes", true);
 
             ResourcesStorageUsedBytes = ydbGroup->GetNamedCounter("name",
                 "resources.storage.used_bytes", false);
@@ -908,6 +930,8 @@ class TTabletMon {
 
             ShardCpuUtilization = ydbGroup->GetNamedHistogram("name",
                 "table.datashard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false);
+            ColumnShardCpuUtilization = ydbGroup->GetNamedHistogram("name",
+                "table.columnshard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false);
         };
 
         void Initialize(::NMonitoring::TDynamicCounterPtr counters, bool hasDatashard, bool hasSchemeshard, bool hasColumnShard) {
@@ -943,8 +967,11 @@ class TTabletMon {
 
                 ColumnShardScannedBytes_ = appGroup->GetCounter("ColumnShard/ScannedBytes");
                 ColumnShardScannedRows_ = appGroup->GetCounter("ColumnShard/ScannedRows");
-                ColumnShardUpsertBlobsWritten_ = appGroup->GetCounter("ColumnShard/UpsertBlobsWritten");
-                ColumnShardUpsertBytesWritten_ = appGroup->GetCounter("ColumnShard/UpsertBytesWritten");
+                ColumnShardOperationsRowsWritten_ = appGroup->GetCounter("ColumnShard/OperationsRowsWritten");
+                ColumnShardOperationsBytesWritten_ = appGroup->GetCounter("ColumnShard/OperationsBytesWritten");
+                ColumnShardErasedBytes_ = appGroup->GetCounter("ColumnShard/BytesErased");
+                ColumnShardErasedRows_ = appGroup->GetCounter("ColumnShard/RowsErased");
+                ColumnShardConsumedCpuHistogram = appGroup->FindHistogram("HIST(ConsumedCPU)");
             }
 
             if (hasSchemeshard && !DiskSpaceTablesTotalBytes) {
@@ -990,10 +1017,20 @@ class TTabletMon {
             }
 
             if (ColumnShardScannedBytes_) {
+                ColumnShardReadRows_->Set(0);
+                ColumnShardReadBytes_->Set(0);
                 ColumnShardScanRows_->Set(ColumnShardScannedRows_->Val());
                 ColumnShardScanBytes_->Set(ColumnShardScannedBytes_->Val());
-                ColumnShardBulkUpsertRows_->Set(ColumnShardUpsertBlobsWritten_->Val());
-                ColumnShardBulkUpsertBytes_->Set(ColumnShardUpsertBytesWritten_->Val());
+                ColumnShardWriteRows_->Set(ColumnShardOperationsRowsWritten_->Val());
+                ColumnShardWriteBytes_->Set(ColumnShardOperationsBytesWritten_->Val());
+                ColumnShardBulkUpsertRows_->Set(ColumnShardOperationsRowsWritten_->Val());
+                ColumnShardBulkUpsertBytes_->Set(ColumnShardOperationsBytesWritten_->Val());
+                ColumnShardEraseRows_->Set(ColumnShardErasedRows_->Val());
+                ColumnShardEraseBytes_->Set(ColumnShardErasedBytes_->Val());
+
+                if (ColumnShardConsumedCpuHistogram) {
+                    TransferBuckets(ColumnShardCpuUtilization, ColumnShardConsumedCpuHistogram);
+                }
             }
 
             if (DiskSpaceTablesTotalBytes) {

ydb/core/tx/columnshard/background_controller.h

@@ -1,6 +1,7 @@
 #pragma once
 #include "engines/changes/abstract/compaction_info.h"
 #include "engines/portions/meta.h"
+#include <ydb/core/tx/columnshard/counters/counters_manager.h>
 
 namespace NKikimr::NOlap {
 class TColumnEngineChanges;
@@ -15,11 +16,16 @@ class TBackgroundController {
     using TCurrentCompaction = THashMap<ui64, NOlap::TPlanCompactionInfo>;
     TCurrentCompaction ActiveCompactionInfo;
 
+    std::shared_ptr<TBackgroundControllerCounters> Counters;
     bool ActiveCleanupPortions = false;
     bool ActiveCleanupTables = false;
     bool ActiveCleanupInsertTable = false;
     YDB_READONLY(TMonotonic, LastIndexationInstant, TMonotonic::Zero());
 public:
+    TBackgroundController(std::shared_ptr<TBackgroundControllerCounters> counters)
+        : Counters(std::move(counters)) {
+    }
+
     THashSet<NOlap::TPortionAddress> GetConflictTTLPortions() const;
     THashSet<NOlap::TPortionAddress> GetConflictCompactionPortions() const;
 
@@ -29,6 +35,7 @@ class TBackgroundController {
     bool StartCompaction(const NOlap::TPlanCompactionInfo& info);
     void FinishCompaction(const NOlap::TPlanCompactionInfo& info) {
         Y_ABORT_UNLESS(ActiveCompactionInfo.erase(info.GetPathId()));
+        Counters->OnCompactionFinish(info.GetPathId());
     }
     const TCurrentCompaction& GetActiveCompaction() const {
         return ActiveCompactionInfo;

ydb/core/tx/columnshard/blobs_action/bs/write.cpp

@@ -15,13 +15,10 @@ void TWriteAction::DoOnCompleteTxAfterWrite(NColumnShard::TColumnShard& self, co
     ui64 blobsWritten = BlobBatch.GetBlobCount();
     ui64 bytesWritten = BlobBatch.GetTotalSize();
     if (blobsWroteSuccessfully) {
-        self.IncCounter(NColumnShard::COUNTER_UPSERT_BLOBS_WRITTEN, blobsWritten);
-        self.IncCounter(NColumnShard::COUNTER_UPSERT_BYTES_WRITTEN, bytesWritten);
-        //    self.IncCounter(NColumnShard::COUNTER_RAW_BYTES_UPSERTED, insertedBytes);
-        self.IncCounter(NColumnShard::COUNTER_WRITE_SUCCESS);
+        self.Counters.GetTabletCounters()->OnWriteSuccess(blobsWritten, bytesWritten);
         Manager->SaveBlobBatchOnComplete(std::move(BlobBatch));
     } else {
-        self.IncCounter(NColumnShard::COUNTER_WRITE_FAIL);
+        self.Counters.GetTabletCounters()->OnWriteFailure();
     }
 }
 

ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp

@@ -70,7 +70,7 @@ bool TTxWrite::Execute(TTransactionContext& txc, const TActorContext&) {
 
             if (!InsertOneBlob(txc, i, writeId)) {
                 LOG_S_DEBUG(TxPrefix() << "duplicate writeId " << (ui64)writeId << TxSuffix());
-                Self->IncCounter(COUNTER_WRITE_DUPLICATE);
+                Self->Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_DUPLICATE);
             }
         }
     }
@@ -140,10 +140,10 @@ void TTxWrite::Complete(const TActorContext& ctx) {
     }
     for (ui32 i = 0; i < buffer.GetAggregations().size(); ++i) {
         const auto& writeMeta = buffer.GetAggregations()[i]->GetWriteMeta();
-        Self->CSCounters.OnWriteTxComplete(now - writeMeta.GetWriteStartInstant());
-        Self->CSCounters.OnSuccessWriteResponse();
+        Self->Counters.GetCSCounters().OnWriteTxComplete(now - writeMeta.GetWriteStartInstant());
+        Self->Counters.GetCSCounters().OnSuccessWriteResponse();
     }
-
+    Self->Counters.GetTabletCounters()->IncCounter(COUNTER_IMMEDIATE_TX_COMPLETED);
 }
 
 }