Improve DeepScrubbing output (#21946)

serbel324 · web-flow · commit 36a3f782753f · 2025-07-30T16:44:04.000+03:00
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
@@ -1322,17 +1322,20 @@ struct TEvBlobStorage {
         TLogoBlobID Id;
         TInstant Deadline;
         NKikimrBlobStorage::EGetHandleClass GetHandleClass;
+        bool SingleLine;    // Print DataInfo in single line
 
         ui32 RestartCounter = 0;
         std::shared_ptr<TExecutionRelay> ExecutionRelay;
 
         TEvCheckIntegrity(
                 const TLogoBlobID& id,
                 TInstant deadline,
-                NKikimrBlobStorage::EGetHandleClass getHandleClass)
+                NKikimrBlobStorage::EGetHandleClass getHandleClass,
+                bool singleLine = false)
             : Id(id)
             , Deadline(deadline)
             , GetHandleClass(getHandleClass)
+            , SingleLine(singleLine)
         {}
 
         TString Print(bool /*isFull*/) const {
@@ -1373,13 +1376,44 @@ struct TEvBlobStorage {
             PS_BLOB_IS_RECOVERABLE = 4,     // blob parts are definitely placed incorrectly or there are missing parts but blob may be recovered
             PS_BLOB_IS_LOST = 5,            // blob is lost/unrecoverable
         };
-        EPlacementStatus PlacementStatus = PS_OK;
+
+        static TString PlacementStatusToString(EPlacementStatus status) {
+            switch (status) {
+                case PS_OK:
+                    return "PS_OK";
+                case PS_REPLICATION_IN_PROGRESS:
+                    return "PS_REPLICATION_IN_PROGRESS";
+                case PS_UNKNOWN:
+                    return "PS_UNKNOWN";
+                case PS_BLOB_IS_RECOVERABLE:
+                    return "PS_BLOB_IS_RECOVERABLE";
+                case PS_BLOB_IS_LOST:
+                    return "PS_BLOB_IS_LOST";
+                default:
+                    return "BAD_PLACEMENT_STATUS";
+            }
+        }
 
         enum EDataStatus {
             DS_OK = 1,      // all data parts contain valid data
             DS_UNKNOWN = 2, // status is unknown because of missing disks or network problems
             DS_ERROR = 3,   // some parts definitely contain invalid data
         };
+
+        static TString DataStatusToString(EDataStatus status) {
+            switch (status) {
+                case DS_OK:
+                    return "DS_OK";
+                case DS_UNKNOWN:
+                    return "DS_UNKNOWN";
+                case DS_ERROR:
+                    return "DS_ERROR";
+                default:
+                    return "BAD_DATA_STATUS";
+            }
+        }
+
+        EPlacementStatus PlacementStatus = PS_OK;
         EDataStatus DataStatus = DS_OK;
         TString DataInfo; // textual info about checks in blob data
 
@@ -1395,8 +1429,8 @@ struct TEvBlobStorage {
                 << " Id# " << Id
                 << " Status# " << NKikimrProto::EReplyStatus_Name(Status)
                 << " ErrorReason# " << ErrorReason
-                << " PlacementStatus# " << (int)PlacementStatus
-                << " DataStatus# " << (int)DataStatus
+                << " PlacementStatus# " << PlacementStatusToString(PlacementStatus)
+                << " DataStatus# " << DataStatusToString(DataStatus)
                 << " DataInfo# " << DataInfo
                 << " }";
             return str.Str();
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_check_integrity_get.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_check_integrity_get.cpp
@@ -14,6 +14,7 @@ class TBlobStorageGroupCheckIntegrityRequest
     const TLogoBlobID Id;
     const TInstant Deadline;
     const NKikimrBlobStorage::EGetHandleClass GetHandleClass;
+    const bool SingleLine;
 
     TGroupQuorumTracker QuorumTracker;
 
@@ -162,8 +163,10 @@ class TBlobStorageGroupCheckIntegrityRequest
             }
         }
 
+        char separator = SingleLine ? ' ' : '\n';
+
         const auto& dataChecker = Info->GetTopology().GetDataIntegrityChecker();
-        auto partsState = dataChecker.GetDataState(Id, PartsData);
+        auto partsState = dataChecker.GetDataState(Id, PartsData, separator);
 
         if (partsState.IsOk) {
             PendingResult->DataStatus = (PendingResult->PlacementStatus == TEvCheckIntegrityResult::PS_UNKNOWN) ?
@@ -173,10 +176,10 @@ class TBlobStorageGroupCheckIntegrityRequest
         }
 
         TStringStream str;
-        str << "Disks:" << Endl;
+        str << "Disks:" << separator;
         for (ui32 diskIdx = 0; diskIdx < Info->Type.BlobSubgroupSize(); ++diskIdx) {
             auto vDiskIdShort = Info->GetTopology().GetVDiskInSubgroup(diskIdx, Id.Hash());
-            str << diskIdx << ": " << Info->CreateVDiskID(vDiskIdShort) << Endl;
+            str << diskIdx << ": " << Info->CreateVDiskID(vDiskIdShort) << separator;
         }
 
         PendingResult->DataInfo = str.Str();
@@ -199,6 +202,7 @@ class TBlobStorageGroupCheckIntegrityRequest
         , Id(params.Common.Event->Id)
         , Deadline(params.Common.Event->Deadline)
         , GetHandleClass(params.Common.Event->GetHandleClass)
+        , SingleLine(params.Common.Event->SingleLine)
         , QuorumTracker(Info.Get())
     {}
 
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h
@@ -172,7 +172,7 @@ class TBlobStorageGroupInfo : public TThrRefBase {
             TString DataInfo;
         };
 
-        virtual TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData) const = 0;
+        virtual TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData, char separator) const = 0;
     };
 
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_data_check.h b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_data_check.h
@@ -16,9 +16,10 @@ class TDataIntegrityCheckerTrivial : public TDataIntegrityCheckerBase {
 public:
     using TDataIntegrityCheckerBase::TDataIntegrityCheckerBase;
 
-    TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData) const override {
+    TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData, char separator) const override {
         Y_UNUSED(id);
         Y_UNUSED(partsData);
+        Y_UNUSED(separator);
         return {};
     }
 };
@@ -27,7 +28,7 @@ class TDataIntegrityCheckerBlock42 : public TDataIntegrityCheckerBase {
 public:
     using TDataIntegrityCheckerBase::TDataIntegrityCheckerBase;
 
-    TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData) const override {
+    TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData, char separator) const override {
         Y_ABORT_UNLESS(partsData.Parts.size() == 6);
 
         TPartsState partsState;
@@ -58,7 +59,7 @@ class TDataIntegrityCheckerBlock42 : public TDataIntegrityCheckerBase {
 
         // checking layout
         TStringStream layoutReport;
-        layoutReport << "Layout info:" << Endl;
+        layoutReport << "Layout info:" << separator;
 
         TStringStream str;
         bool hasUnequalParts = false;
@@ -80,19 +81,19 @@ class TDataIntegrityCheckerBlock42 : public TDataIntegrityCheckerBase {
                 str << "]";
                 ++ver;
             }
-            str << Endl;
+            str << separator;
         }
 
         layoutReport << str.Str();
         if (hasUnequalParts) {
             partsState.IsOk = false;
-            layoutReport << "ERROR: There are unequal parts" << Endl;
+            layoutReport << "ERROR: There are unequal parts" << separator;
         }
         partsState.DataInfo = layoutReport.Str();
 
         // checking erasure
         TStringStream erasureReport;
-        erasureReport << "Erasure info:" << Endl;
+        erasureReport << "Erasure info:" << separator;
 
         std::vector<ui32> partIds;
         partIds.reserve(6);
@@ -144,7 +145,7 @@ class TDataIntegrityCheckerBlock42 : public TDataIntegrityCheckerBase {
                         if (cmp) {
                             erasureError = true;
                         } else {
-                            str << "OK" << Endl;
+                            str << "OK" << separator;
                             erasureReport << str.Str(); // report only succesful restore
                         }
                     }
@@ -199,7 +200,7 @@ class TDataIntegrityCheckerBlock42 : public TDataIntegrityCheckerBase {
 
         if (erasureError) {
             partsState.IsOk = false;
-            erasureReport << "ERROR: There are erasure restore fails" << Endl;
+            erasureReport << "ERROR: There are erasure restore fails" << separator;
         }
 
         partsState.DataInfo += erasureReport.Str();
@@ -214,7 +215,7 @@ class TDataIntegrityCheckerMirror : public TDataIntegrityCheckerBase {
 public:
     using TDataIntegrityCheckerBase::TDataIntegrityCheckerBase;
 
-    TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData) const override {
+    TPartsState GetDataState(const TLogoBlobID& id, const TPartsData& partsData, char separator) const override {
         Y_UNUSED(id);
         Y_ABORT_UNLESS(partsData.Parts.size() == 3);
 
@@ -244,7 +245,7 @@ class TDataIntegrityCheckerMirror : public TDataIntegrityCheckerBase {
         }
 
         TStringStream layoutReport;
-        layoutReport << "Layout info:" << Endl;
+        layoutReport << "Layout info:" << separator;
 
         TStringStream str;
         bool hasUnequalParts = (seenParts.size() > 1);
@@ -260,12 +261,12 @@ class TDataIntegrityCheckerMirror : public TDataIntegrityCheckerBase {
             str << "]";
             ++ver;
         }
-        str << Endl;
+        str << separator;
         layoutReport << str.Str();
 
         if (hasUnequalParts) {
             partsState.IsOk = false;
-            layoutReport << "ERROR: There are unequal parts" << Endl;
+            layoutReport << "ERROR: There are unequal parts" << separator;
         }
         partsState.DataInfo = layoutReport.Str();
 
diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h
@@ -958,7 +958,32 @@ struct TEnvironmentSetup {
             }
         }
         return ctr;
-    };
+    }
+
+    template <class TCallback>
+    ui64 AggregateVDiskCountersWithCallback(TString storagePool, ui32 nodesCount, ui32 groupSize, ui32 groupId,
+            const std::vector<ui32>& pdiskLayout, TCallback callback) {
+        ui64 ctr = 0;
+
+        for (ui32 nodeId = 1; nodeId <= nodesCount; ++nodeId) {
+            auto* appData = Runtime->GetNode(nodeId)->AppData.get();
+            for (ui32 i = 0; i < groupSize; ++i) {
+                TStringStream ss;
+                ss << LeftPad(i, 2, '0');
+                TString orderNumber = ss.Str();
+                ss.Clear();
+                ss << LeftPad(pdiskLayout[i], 9, '0');
+                TString pdisk = ss.Str();
+                ctr += callback(GetServiceCounters(appData->Counters, "vdisks")->
+                        GetSubgroup("storagePool", storagePool)->
+                        GetSubgroup("group", std::to_string(groupId))->
+                        GetSubgroup("orderNumber", orderNumber)->
+                        GetSubgroup("pdisk", pdisk)->
+                        GetSubgroup("media", "rot"));
+            }
+        }
+        return ctr;
+    }
 
     void SetIcbControl(ui32 nodeId, TString controlName, ui64 value) {
         if (nodeId == 0) {
diff --git a/ydb/core/blobstorage/ut_blobstorage/scrub.cpp b/ydb/core/blobstorage/ut_blobstorage/scrub.cpp
@@ -433,6 +433,25 @@ Y_UNIT_TEST_SUITE(DeepScrubbing) {
             , PartCorruptionMask(partCorruptionMask)
         {}
 
+        struct TAggregateScrubMetrics {
+            TAggregateScrubMetrics(TString counterName, bool isHuge, TErasureType::EErasureSpecies erasure)
+                : CounterName(counterName)
+                , IsHuge(isHuge)
+                , Erasure(erasure)
+            {}
+
+            ui64 operator()(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) const {
+                return counters->GetSubgroup("subsystem", "deepScrubbing")
+                        ->GetSubgroup("blobSize", IsHuge ? "huge" : "small")
+                        ->GetSubgroup("erasure", TErasureType::ErasureSpeciesName(Erasure))
+                        ->GetCounter(CounterName, true)->Val();
+            }
+
+            TString CounterName;
+            bool IsHuge;
+            TErasureType::EErasureSpecies Erasure;
+        };
+
         void RunTest() {
             Initialize();
             AllocateEdgeActor(true);
@@ -506,19 +525,16 @@ Y_UNIT_TEST_SUITE(DeepScrubbing) {
                 }
             }
 
+            bool isHuge = (BlobSize == EBlobSize::Val_HugeBlob);
+
             std::vector<ui32> pdiskLayout = MakePDiskLayout(BaseConfig, groupInfo->GetTopology(), GroupId);
 
             ui64 blobsScrubbed =
-                    Env->AggregateVDiskCounters(Env->StoragePoolName, NodeCount, Erasure.BlobSubgroupSize(),
-                            GroupId, pdiskLayout, "deepScrubbing", "SmallBlobsChecked", false) +
-                    Env->AggregateVDiskCounters(Env->StoragePoolName, NodeCount, Erasure.BlobSubgroupSize(),
-                            GroupId, pdiskLayout, "deepScrubbing", "HugeBlobsChecked", false);
-
+                    Env->AggregateVDiskCountersWithCallback(Env->StoragePoolName, NodeCount, Erasure.BlobSubgroupSize(),
+                            GroupId, pdiskLayout, TAggregateScrubMetrics("BlobsChecked", isHuge, Erasure.GetErasure()));
             ui64 dataIssues =
-                    Env->AggregateVDiskCounters(Env->StoragePoolName, NodeCount, Erasure.BlobSubgroupSize(),
-                            GroupId, pdiskLayout, "deepScrubbing", "DataIssuesSmallBlobs", false) +
-                    Env->AggregateVDiskCounters(Env->StoragePoolName, NodeCount, Erasure.BlobSubgroupSize(),
-                            GroupId, pdiskLayout, "deepScrubbing", "DataIssuesHugeBlobs", false);
+                    Env->AggregateVDiskCountersWithCallback(Env->StoragePoolName, NodeCount, Erasure.BlobSubgroupSize(),
+                            GroupId, pdiskLayout, TAggregateScrubMetrics("DataIssues", isHuge, Erasure.GetErasure()));
 
             UNIT_ASSERT_VALUES_UNEQUAL_C(blobsScrubbed, 0, makePrefix());
             UNIT_ASSERT_VALUES_UNEQUAL_C(dataIssues, 0, makePrefix()
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h b/ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h
@@ -676,33 +676,60 @@ public:
             COUNTER_DEF(BlobsFixed);
         };
 
+        ///////////////////////////////////////////////////////////////////////////////////
+        // TDeepScrubbingGroup
+        ///////////////////////////////////////////////////////////////////////////////////
         class TDeepScrubbingGroup : public TBase {
         public:
             GROUP_CONSTRUCTOR(TDeepScrubbingGroup)
             {
-                COUNTER_INIT(SmallBlobsChecked, true);
-                COUNTER_INIT(HugeBlobsChecked, true);
-                COUNTER_INIT(CheckIntegritySuccesses, false);
-                COUNTER_INIT(CheckIntegrityErrors, false);
-
-                COUNTER_INIT(PlacementIssuesSmallBlobs, false);
-                COUNTER_INIT(DataIssuesSmallBlobs, false);
-
-                COUNTER_INIT(PlacementIssuesHugeBlobs, false);
-                COUNTER_INIT(DataIssuesHugeBlobs, false);
+                COUNTER_INIT(BlobsChecked, true);
+                COUNTER_INIT(CheckIntegritySuccesses, true);
+                COUNTER_INIT(CheckIntegrityErrors, true);
+                COUNTER_INIT(UnknownDataStatus, true);
+                COUNTER_INIT(UnknownPlacementStatus, true);
+                COUNTER_INIT(DataIssues, true);
+                COUNTER_INIT(PlacementIssues, true);
             }
 
-            COUNTER_DEF(SmallBlobsChecked);
-            COUNTER_DEF(HugeBlobsChecked);
-
+            COUNTER_DEF(BlobsChecked);
             COUNTER_DEF(CheckIntegritySuccesses);
             COUNTER_DEF(CheckIntegrityErrors);
+            COUNTER_DEF(UnknownDataStatus);
+            COUNTER_DEF(UnknownPlacementStatus);
+            COUNTER_DEF(DataIssues);
+            COUNTER_DEF(PlacementIssues);
+        };
 
-            COUNTER_DEF(PlacementIssuesSmallBlobs);
-            COUNTER_DEF(DataIssuesSmallBlobs);
+        class TDeepScrubbingSubgroups {
+        public:
+            TDeepScrubbingSubgroups(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) {
+                for (bool isHuge : {true, false}) {
+                    for (TErasureType::EErasureSpecies erasure :
+                            {TErasureType::ErasureNone, TErasureType::Erasure4Plus2Block,
+                            TErasureType::ErasureMirror3of4, TErasureType::ErasureMirror3dc}) {
+                        ::NMonitoring::TDynamicCounterPtr subgroup = counters
+                                ->GetSubgroup("blobSize", isHuge ? "huge" : "small")
+                                ->GetSubgroup("erasure", TErasureType::ErasureSpeciesName(erasure));
+                        Subgroups.insert({GetKey(isHuge, erasure), TDeepScrubbingGroup(subgroup)});
+                    }
+                }
+            }
+
+            TDeepScrubbingGroup* GetCounters(bool isHuge, TErasureType::EErasureSpecies erasure) {
+                auto it = Subgroups.find(GetKey(isHuge, erasure));
+                if (it == Subgroups.end()) {
+                    return nullptr;
+                }
+                return &it->second;
+            }
 
-            COUNTER_DEF(PlacementIssuesHugeBlobs);
-            COUNTER_DEF(DataIssuesHugeBlobs);
+        private:
+            std::unordered_map<ui64, TDeepScrubbingGroup> Subgroups;
+
+            ui64 GetKey(bool isHuge, TErasureType::EErasureSpecies erasure) {
+                return ((ui64)isHuge << 32) + (ui64)erasure;
+            }
         };
 
     } // NMonGroup
diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp b/ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp
diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_impl.h b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_impl.h
