storage healthceck fixes (#8460)

vporyadke · alexvru · web-flow · commit c9f4e8145463 · 2024-08-29T16:48:27.000+02:00
Co-authored-by: Alexander Rutkovsky &lt;alexvru@ydb.tech&gt;
diff --git a/ydb/core/health_check/health_check.cpp b/ydb/core/health_check/health_check.cpp
@@ -249,6 +249,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
     struct TGroupState {
         TString ErasureSpecies;
         std::vector<const NKikimrSysView::TVSlotEntry*> VSlots;
+        ui32 Generation;
     };
 
     struct TSelfCheckResult {
@@ -1288,12 +1289,17 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
         for (const auto& group : Groups->GetEntries()) {
             auto groupId = group.GetKey().GetGroupId();
             auto poolId = group.GetInfo().GetStoragePoolId();
-            GroupState[groupId].ErasureSpecies = group.GetInfo().GetErasureSpeciesV2();
+            auto& groupState = GroupState[groupId];
+            groupState.ErasureSpecies = group.GetInfo().GetErasureSpeciesV2();
+            groupState.Generation = group.GetInfo().GetGeneration();
             StoragePoolState[poolId].Groups.emplace(groupId);
         }
         for (const auto& vSlot : VSlots->GetEntries()) {
             auto vSlotId = GetVSlotId(vSlot.GetKey());
-            GroupState[vSlot.GetInfo().GetGroupId()].VSlots.push_back(&vSlot);
+            auto groupStateIt = GroupState.find(vSlot.GetInfo().GetGroupId());
+            if (groupStateIt != GroupState.end() && vSlot.GetInfo().GetGroupGeneration() == groupStateIt->second.Generation) {
+                groupStateIt->second.VSlots.push_back(&vSlot);
+            }
         }
         for (const auto& pool : StoragePools->GetEntries()) { // there is no specific pool for static group here
             ui64 poolId = pool.GetKey().GetStoragePoolId();
@@ -1797,6 +1803,13 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
 
         storageVDiskStatus.set_id(GetVSlotId(vSlot->GetKey()));
 
+        if (!vSlot->GetInfo().HasStatusV2()) {
+            // this should mean that BSC recently restarted and does not have accurate data yet - we should not report to avoid false positives
+            context.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN);
+            storageVDiskStatus.set_overall(context.GetOverallStatus());
+            return;
+        }
+
         const auto *descriptor = NKikimrBlobStorage::EVDiskStatus_descriptor();
         auto status = descriptor->FindValueByName(vSlot->GetInfo().GetStatusV2());
         if (!status) { // this case is not expected because becouse bsc assignes status according EVDiskStatus enum
@@ -1816,16 +1829,12 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
                 storageVDiskStatus.set_overall(context.GetOverallStatus());
                 return;
             }
-            case NKikimrBlobStorage::INIT_PENDING: { // initialization in process
-                context.ReportStatus(Ydb::Monitoring::StatusFlag::YELLOW, TStringBuilder() << "VDisk is being initialized", ETags::VDiskState);
-                storageVDiskStatus.set_overall(context.GetOverallStatus());
-                return;
-            }
             case NKikimrBlobStorage::REPLICATING: { // the disk accepts queries, but not all the data was replicated
                 context.ReportStatus(Ydb::Monitoring::StatusFlag::BLUE, TStringBuilder() << "Replication in progress", ETags::VDiskState);
                 storageVDiskStatus.set_overall(context.GetOverallStatus());
                 return;
             }
+            case NKikimrBlobStorage::INIT_PENDING:
             case NKikimrBlobStorage::READY: { // the disk is fully operational and does not affect group fault tolerance
                 context.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN);
             }
@@ -1999,9 +2008,9 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
 
         switch (vDiskInfo.GetVDiskState()) {
             case NKikimrWhiteboard::EVDiskState::OK:
+            case NKikimrWhiteboard::EVDiskState::Initial:
                 context.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN);
                 break;
-            case NKikimrWhiteboard::EVDiskState::Initial:
             case NKikimrWhiteboard::EVDiskState::SyncGuidRecovery:
                 context.IssueRecords.clear();
                 context.ReportStatus(Ydb::Monitoring::StatusFlag::YELLOW,
diff --git a/ydb/core/health_check/health_check_ut.cpp b/ydb/core/health_check/health_check_ut.cpp
diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp
@@ -337,7 +337,7 @@ void TBlobStorageController::ValidateInternalState() {
             Y_ABORT_UNLESS(donor->GetShortVDiskId() == vslot->GetShortVDiskId());
         }
         if (vslot->Group) {
-            if (vslot->Status == NKikimrBlobStorage::EVDiskStatus::READY) {
+            if (vslot->GetStatus() == NKikimrBlobStorage::EVDiskStatus::READY) {
                 Y_DEBUG_ABORT_UNLESS(vslot->IsReady || vslot->IsInVSlotReadyTimestampQ());
             } else {
                 Y_DEBUG_ABORT_UNLESS(!vslot->IsReady && !vslot->IsInVSlotReadyTimestampQ());
@@ -401,7 +401,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) {
             const auto& record = msg->Record;
             for (const auto& item : record.GetVDiskStatus()) {
                 const TVSlotId vslotId(item.GetNodeId(), item.GetPDiskId(), item.GetVSlotId());
-                if (TVSlotInfo *slot = FindVSlot(vslotId); slot && slot->Status > item.GetStatus()) {
+                if (TVSlotInfo *slot = FindVSlot(vslotId); slot && slot->GetStatus() > item.GetStatus()) {
                     return 1;
                 } else if (const auto it = StaticVSlots.find(vslotId); it != StaticVSlots.end() && it->second.VDiskStatus > item.GetStatus()) {
                     return 1;
diff --git a/ydb/core/mind/bscontroller/cmds_box.cpp b/ydb/core/mind/bscontroller/cmds_box.cpp
@@ -213,7 +213,7 @@ namespace NKikimr::NBsController {
         for (const auto& [id, slot] : pdisk->VSlotsOnPDisk) {
             if (slot->Group) {
                 auto *m = VSlots.FindForUpdate(slot->VSlotId);
-                m->Status = NKikimrBlobStorage::EVDiskStatus::ERROR;
+                m->VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR;
                 m->IsReady = false;
                 TGroupInfo *group = Groups.FindForUpdate(slot->Group->ID);
                 GroupFailureModelChanged.insert(slot->Group->ID);
diff --git a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp
@@ -557,7 +557,7 @@ namespace NKikimr::NBsController {
                     x->MutableVDiskMetrics()->CopyFrom(*vslot.VDiskMetrics);
                     x->MutableVDiskMetrics()->ClearVDiskId();
                 }
-                x->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.VDiskStatus));
+                x->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.VDiskStatus.value_or(NKikimrBlobStorage::EVDiskStatus::ERROR)));
                 x->SetReady(vslot.ReadySince <= mono);
             }
             if (const auto& s = Self.StorageConfig; s.HasBlobStorageConfig()) {
@@ -698,7 +698,7 @@ namespace NKikimr::NBsController {
 
         TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId);
         vslot->Mood = TMood::Wipe;
-        vslot->Status = NKikimrBlobStorage::EVDiskStatus::ERROR;
+        vslot->VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR;
         vslot->IsReady = false;
         GroupFailureModelChanged.insert(group->ID);
         group->CalculateGroupStatus();
@@ -744,7 +744,7 @@ namespace NKikimr::NBsController {
 
         TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId);
         vslot->Mood = targetMood;
-        vslot->Status = NKikimrBlobStorage::EVDiskStatus::ERROR;
+        vslot->VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR;
         vslot->IsReady = false;
         GroupFailureModelChanged.insert(group->ID);
         group->CalculateGroupStatus();
diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp
@@ -486,9 +486,9 @@ namespace NKikimr::NBsController {
                 if (!overlay->second || !overlay->second->Group) { // deleted one
                     (overlay->second ? overlay->second : base->second)->DropFromVSlotReadyTimestampQ();
                     NotReadyVSlotIds.erase(overlay->first);
-                } else if (overlay->second->Status != NKikimrBlobStorage::EVDiskStatus::READY) {
+                } else if (overlay->second->GetStatus() != NKikimrBlobStorage::EVDiskStatus::READY) {
                     overlay->second->DropFromVSlotReadyTimestampQ();
-                } else if (!base || base->second->Status != NKikimrBlobStorage::EVDiskStatus::READY) {
+                } else if (!base || base->second->GetStatus() != NKikimrBlobStorage::EVDiskStatus::READY) {
                     overlay->second->PutInVSlotReadyTimestampQ(now);
                 } else {
                     Y_DEBUG_ABORT_UNLESS(overlay->second->IsReady || overlay->second->IsInVSlotReadyTimestampQ());
@@ -998,7 +998,7 @@ namespace NKikimr::NBsController {
             pb->SetAllocatedSize(vslot.Metrics.GetAllocatedSize());
             pb->MutableVDiskMetrics()->CopyFrom(vslot.Metrics);
             pb->MutableVDiskMetrics()->ClearVDiskId();
-            pb->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.Status));
+            pb->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.GetStatus()));
             for (const TVSlotId& vslotId : vslot.Donors) {
                 auto *item = pb->AddDonors();
                 Serialize(item->MutableVSlotId(), vslotId);
diff --git a/ydb/core/mind/bscontroller/config_fit_groups.cpp b/ydb/core/mind/bscontroller/config_fit_groups.cpp
@@ -524,7 +524,7 @@ namespace NKikimr {
                         // also we have to find replicating VSlots on this PDisk and assume they consume up to
                         // max(vslotSize for every slot in group), not their actual AllocatedSize
                         for (const auto& [id, slot] : info.VSlotsOnPDisk) {
-                            if (slot->Group && slot->Status != NKikimrBlobStorage::EVDiskStatus::READY) {
+                            if (slot->Group && slot->GetStatus() != NKikimrBlobStorage::EVDiskStatus::READY) {
                                 ui64 maxGroupSlotSize = 0;
                                 for (const TVSlotInfo *peer : slot->Group->VDisksInGroup) {
                                     maxGroupSlotSize = Max(maxGroupSlotSize, peer->Metrics.GetAllocatedSize());
diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h
@@ -124,16 +124,17 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
         TVSlotReadyTimestampQ::iterator VSlotReadyTimestampIter;
 
     public:
-        NKikimrBlobStorage::EVDiskStatus Status = NKikimrBlobStorage::EVDiskStatus::ERROR;
+        std::optional<NKikimrBlobStorage::EVDiskStatus> VDiskStatus;
+        NHPTimer::STime VDiskStatusTimestamp = GetCycleCountFast();
         bool IsReady = false;
         bool OnlyPhantomsRemain = false;
 
     public:
         void SetStatus(NKikimrBlobStorage::EVDiskStatus status, TMonotonic now, TInstant instant, bool onlyPhantomsRemain) {
-            if (status != Status) {
+            if (status != VDiskStatus) {
                 if (status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // became "replicating"
                     LastGotReplicating = instant;
-                } else if (Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // was "replicating"
+                } else if (VDiskStatus == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // was "replicating"
                     Y_DEBUG_ABORT_UNLESS(LastGotReplicating != TInstant::Zero());
                     ReplicationTime += instant - LastGotReplicating;
                     LastGotReplicating = {};
@@ -145,7 +146,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
                     LastSeenReady = instant;
                 }
 
-                Status = status;
+                VDiskStatus = status;
                 IsReady = false;
                 if (status == NKikimrBlobStorage::EVDiskStatus::READY) {
                     PutInVSlotReadyTimestampQ(now);
@@ -159,6 +160,10 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
             }
         }
 
+        NKikimrBlobStorage::EVDiskStatus GetStatus() const {
+            return VDiskStatus.value_or(NKikimrBlobStorage::EVDiskStatus::ERROR);
+        }
+
         void PutInVSlotReadyTimestampQ(TMonotonic now) {
             const TMonotonic readyAfter = now + ReadyStablePeriod; // vdisk will be treated as READY one shortly, but not now
             Y_ABORT_UNLESS(VSlotReadyTimestampIter == TVSlotReadyTimestampQ::iterator());
@@ -291,15 +296,16 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
 
         TString GetStatusString() const {
             TStringStream s;
-            s << NKikimrBlobStorage::EVDiskStatus_Name(Status);
-            if (Status == NKikimrBlobStorage::REPLICATING && OnlyPhantomsRemain) {
+            const auto status = GetStatus();
+            s << NKikimrBlobStorage::EVDiskStatus_Name(status);
+            if (status == NKikimrBlobStorage::REPLICATING && OnlyPhantomsRemain) {
                 s << "/p";
             }
             return s.Str();
         }
 
         bool IsOperational() const {
-            return Status >= NKikimrBlobStorage::REPLICATING;
+            return GetStatus() >= NKikimrBlobStorage::REPLICATING;
         }
 
         void OnCommit();
@@ -2276,7 +2282,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
                 histo.IncrementFor(passed.Seconds());
 
                 TDuration timeBeingReplicating = slot->ReplicationTime;
-                if (slot->Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) {
+                if (slot->GetStatus() == NKikimrBlobStorage::EVDiskStatus::REPLICATING) {
                     timeBeingReplicating += now - slot->LastGotReplicating;
                 }
 
@@ -2301,7 +2307,8 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
         const NKikimrBlobStorage::TVDiskKind::EVDiskKind VDiskKind;
 
         std::optional<NKikimrBlobStorage::TVDiskMetrics> VDiskMetrics;
-        NKikimrBlobStorage::EVDiskStatus VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR;
+        std::optional<NKikimrBlobStorage::EVDiskStatus> VDiskStatus;
+        NHPTimer::STime VDiskStatusTimestamp = GetCycleCountFast();
         TMonotonic ReadySince = TMonotonic::Max(); // when IsReady becomes true for this disk; Max() in non-READY state
 
         TStaticVSlotInfo(const NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk& vdisk,
@@ -2315,6 +2322,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
                 TStaticVSlotInfo& item = it->second;
                 VDiskMetrics = std::move(item.VDiskMetrics);
                 VDiskStatus = item.VDiskStatus;
+                VDiskStatusTimestamp = item.VDiskStatusTimestamp;
                 ReadySince = item.ReadySince;
             }
         }
diff --git a/ydb/core/mind/bscontroller/monitoring.cpp b/ydb/core/mind/bscontroller/monitoring.cpp
@@ -1296,7 +1296,7 @@ void TBlobStorageController::RenderVSlotRow(IOutputStream& out, const TVSlotInfo
             }
             TABLED() {
                 TDuration time = vslot.ReplicationTime;
-                if (vslot.Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) {
+                if (vslot.GetStatus() == NKikimrBlobStorage::EVDiskStatus::REPLICATING) {
                     time += TActivationContext::Now() - vslot.LastGotReplicating;
                 }
                 out << time;
diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp
@@ -547,7 +547,7 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId, TActorId serve
             updates.push_back({
                 .VDiskId = it->second->GetVDiskId(),
                 .IsReady = it->second->IsReady,
-                .VDiskStatus = it->second->Status,
+                .VDiskStatus = it->second->GetStatus(),
             });
             ScrubState.UpdateVDiskState(&*it->second);
             SysViewChangedVSlots.insert(it->second->VSlotId);
diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp
@@ -913,7 +913,7 @@ namespace NKikimr::NBsController {
                     slot->OnlyPhantomsRemain,
                     slot->IsReady,
                     TMonotonic::Zero(),
-                    slot->Status,
+                    slot->GetStatus(),
                 };
             }
         }
@@ -960,7 +960,7 @@ namespace NKikimr::NBsController {
                         false, /* OnlyPhantomsRemain */
                         true, /* IsReady; decision is based on ReadySince */
                         info.ReadySince,
-                        info.VDiskStatus,
+                        info.VDiskStatus.value_or(NKikimrBlobStorage::EVDiskStatus::ERROR),
                     };
                 }
             }
@@ -987,7 +987,7 @@ namespace NKikimr::NBsController {
                 const bool was = slot->IsOperational();
                 if (const TGroupInfo *group = slot->Group) {
                     const bool wasReady = slot->IsReady;
-                    if (slot->Status != m.GetStatus() || slot->OnlyPhantomsRemain != m.GetOnlyPhantomsRemain()) {
+                    if (slot->GetStatus() != m.GetStatus() || slot->OnlyPhantomsRemain != m.GetOnlyPhantomsRemain()) {
                         slot->SetStatus(m.GetStatus(), mono, now, m.GetOnlyPhantomsRemain());
                         if (slot->IsReady != wasReady) {
                             ScrubState.UpdateVDiskState(slot);
@@ -1001,14 +1001,14 @@ namespace NKikimr::NBsController {
                         .VDiskId = vdiskId,
                         .OnlyPhantomsRemain = slot->OnlyPhantomsRemain,
                         .IsReady = slot->IsReady,
-                        .VDiskStatus = slot->Status,
+                        .VDiskStatus = slot->GetStatus(),
                     });
                     if (!was && slot->IsOperational() && !group->SeenOperational) {
                         groups.insert(const_cast<TGroupInfo*>(group));
                     }
                     SysViewChangedVSlots.insert(vslotId);
                 }
-                if (slot->Status == NKikimrBlobStorage::EVDiskStatus::READY) {
+                if (slot->GetStatus() == NKikimrBlobStorage::EVDiskStatus::READY) {
                     // we can release donor slots without further notice then the VDisk is completely replicated; we
                     // intentionally use GetStatus() here instead of IsReady() to prevent waiting
                     for (const TVSlotId& donorVSlotId : slot->Donors) {
diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp
diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp

Original file line number	Diff line number	Diff line change
`@@ -1296,7 +1296,7 @@ void TBlobStorageController::RenderVSlotRow(IOutputStream& out, const TVSlotInfo`
`1296`	`1296`	`}`
`1297`	`1297`	`TABLED() {`
`1298`	`1298`	`TDuration time = vslot.ReplicationTime;`
`1299`		`- if (vslot.Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) {`
	`1299`	`+ if (vslot.GetStatus() == NKikimrBlobStorage::EVDiskStatus::REPLICATING) {`
`1300`	`1300`	`time += TActivationContext::Now() - vslot.LastGotReplicating;`
`1301`	`1301`	`}`
`1302`	`1302`	`out << time;`