Add logging and monitoring for OOS in vdisk

ydb/core/blobstorage/vdisk/common/blobstorage_status.cpp

@@ -36,7 +36,7 @@ namespace NKikimr {
                 SetRacingGroupInfo(record, Result->Record, GroupInfo);
                 LOG_DEBUG(ctx, BS_VDISK_OTHER, VDISKP(VCtx->VDiskLogPrefix, "TEvVStatusResult Request# {%s} Response# {%s}",
                     SingleLineProto(record).data(), SingleLineProto(Result->Record).data()));
-                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel());
+                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel(), VCtx->VDiskLogPrefix, VCtx->OOSMonGroup);
                 Die(ctx);
                 return;
             }
@@ -72,7 +72,7 @@ namespace NKikimr {
                 ctx.Send(NotifyId, new TEvents::TEvActorDied());
                 LOG_DEBUG(ctx, BS_VDISK_GET,
                     VDISKP(VCtx->VDiskLogPrefix, "TEvVStatusResult"));
-                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel());
+                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel(), VCtx->VDiskLogPrefix, VCtx->OOSMonGroup);
                 Die(ctx);
             }
         }

ydb/core/blobstorage/vdisk/common/vdisk_context.cpp

@@ -40,6 +40,7 @@ namespace NKikimr {
         , VDiskMemCounters(vdiskCounters->GetSubgroup("subsystem", "memhull"))
         , Histograms(VDiskCounters, type)
         , IFaceMonGroup(std::make_shared<NMonGroup::TVDiskIFaceGroup>(VDiskCounters, "subsystem", "interface"))
+        , OOSMonGroup(std::make_shared<NMonGroup::TOutOfSpaceGroup>(VDiskCounters, "subsystem", "oos"))
         , GroupId(selfVDisk.GroupID)
         , ShortSelfVDisk(selfVDisk)
         , VDiskLogPrefix(GenerateVDiskLogPrefix(selfVDisk, donorMode))

ydb/core/blobstorage/vdisk/common/vdisk_context.h

@@ -42,6 +42,7 @@ namespace NKikimr {
         // latency histograms
         NVDiskMon::THistograms Histograms;
         std::shared_ptr<NMonGroup::TVDiskIFaceGroup> IFaceMonGroup;
+        std::shared_ptr<NMonGroup::TOutOfSpaceGroup> OOSMonGroup;
         // Self VDisk related info
         const ui32 GroupId;
         const TVDiskIdShort ShortSelfVDisk;

ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h

@@ -561,6 +561,29 @@ public:
             }
         };
 
+        ///////////////////////////////////////////////////////////////////////////////////
+        // TOutOfSpaceGroup
+        ///////////////////////////////////////////////////////////////////////////////////
+        class TOutOfSpaceGroup : public TBase {
+        public:
+            GROUP_CONSTRUCTOR(TOutOfSpaceGroup)
+            {
+                COUNTER_INIT(ResponsesWithDiskSpaceRed, true);
+                COUNTER_INIT(ResponsesWithDiskSpaceOrange, true);
+                COUNTER_INIT(ResponsesWithDiskSpacePreOrange, true);
+                COUNTER_INIT(ResponsesWithDiskSpaceLightOrange, true);
+                COUNTER_INIT(ResponsesWithDiskSpaceYellowStop, true);
+                COUNTER_INIT(ResponsesWithDiskSpaceLightYellowMove, true);
+            }
+
+            COUNTER_DEF(ResponsesWithDiskSpaceRed);
+            COUNTER_DEF(ResponsesWithDiskSpaceOrange);
+            COUNTER_DEF(ResponsesWithDiskSpacePreOrange);
+            COUNTER_DEF(ResponsesWithDiskSpaceLightOrange);
+            COUNTER_DEF(ResponsesWithDiskSpaceYellowStop);
+            COUNTER_DEF(ResponsesWithDiskSpaceLightYellowMove);
+        };
+
     } // NMonGroup
 } // NKikimr
 

ydb/core/blobstorage/vdisk/common/vdisk_response.cpp

@@ -1,19 +1,40 @@
 #include "vdisk_response.h"
 #include "vdisk_events.h"
 #include <ydb/core/base/interconnect_channels.h>
+#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_space_color.h>
 #include <util/system/datetime.h>
 
 namespace NKikimr {
 
-void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie) {
+void LogOOSStatus(ui32 flags, const TLogoBlobID& blobId, const TString& vDiskLogPrefix);
+void UpdateMonOOSStatus(ui32 flags, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup);
+
+void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup) {
     ui32 channel = TInterconnectChannels::IC_BLOBSTORAGE;
     if (TEvVResultBase *base = dynamic_cast<TEvVResultBase *>(ev)) {
         channel = base->GetChannelToSend();
     }
-    SendVDiskResponse(ctx, recipient, ev, cookie, channel);
+    SendVDiskResponse(ctx, recipient, ev, cookie, channel, vDiskLogPrefix, monGroup);
 }
 
-void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel) {
+void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup) {
+    switch(ev->Type()) {
+        case TEvBlobStorage::TEvVPutResult::EventType: {
+            TEvBlobStorage::TEvVPutResult* event = static_cast<TEvBlobStorage::TEvVPutResult *>(ev);
+            LogOOSStatus(event->Record.GetStatusFlags(), LogoBlobIDFromLogoBlobID(event->Record.GetBlobID()), vDiskLogPrefix);
+            UpdateMonOOSStatus(event->Record.GetStatusFlags(), monGroup);
+            break;
+        }
+        case TEvBlobStorage::TEvVMultiPutResult::EventType: {
+            TEvBlobStorage::TEvVMultiPutResult *event = static_cast<TEvBlobStorage::TEvVMultiPutResult *>(ev);
+            for (ui64 i = 0; i < event->Record.ItemsSize(); ++i) {
+                const auto& item = event->Record.GetItems(i);
+                LogOOSStatus(item.GetStatusFlags(), LogoBlobIDFromLogoBlobID(item.GetBlobID()), vDiskLogPrefix);
+                UpdateMonOOSStatus(item.GetStatusFlags(), monGroup);
+            }
+        }
+    }
+
     switch (const ui32 type = ev->Type()) {
 #define HANDLE_EVENT(T) \
         case TEvBlobStorage::T::EventType: { \
@@ -45,4 +66,34 @@ void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEve
     }
 }
 
+void LogOOSStatus(ui32 flags, const TLogoBlobID& blobId, const TString& vDiskLogPrefix) {
+    if (!TlsActivationContext) {
+        return;
+    }
+
+    LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_VDISK_CHUNKS,
+        vDiskLogPrefix << "Disk space status changed to " <<
+        TPDiskSpaceColor_Name(StatusFlagToSpaceColor(flags)) << " on blob " << blobId.ToString());
+}
+
+void UpdateMonOOSStatus(ui32 flags, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup) {
+    if (!monGroup) {
+        return;
+    }
+
+    if (flags & NKikimrBlobStorage::StatusDiskSpaceRed) {
+        monGroup->ResponsesWithDiskSpaceRed().Inc();
+    } else if (flags & NKikimrBlobStorage::StatusDiskSpaceOrange) {
+        monGroup->ResponsesWithDiskSpaceOrange().Inc();
+    } else if (flags & NKikimrBlobStorage::StatusDiskSpaceLightOrange) {
+        monGroup->ResponsesWithDiskSpaceLightOrange().Inc();
+    } else if (flags & NKikimrBlobStorage::StatusDiskSpacePreOrange) {
+        monGroup->ResponsesWithDiskSpacePreOrange().Inc();
+    } else if (flags & NKikimrBlobStorage::StatusDiskSpaceYellowStop) {
+        monGroup->ResponsesWithDiskSpaceYellowStop().Inc();
+    } else if (flags & NKikimrBlobStorage::StatusDiskSpaceLightYellowMove) {
+        monGroup->ResponsesWithDiskSpaceLightYellowMove().Inc();
+    }
+}
+
 }//NKikimr

ydb/core/blobstorage/vdisk/common/vdisk_response.h

@@ -1,10 +1,11 @@
 #pragma once
 #include "defs.h"
+#include "vdisk_mongroups.h"
 
 namespace NKikimr {
 
-void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie);
+void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup);
 
-void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel);
+void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup);
 
 }//NKikimr

ydb/core/blobstorage/vdisk/query/query_barrier.cpp

@@ -33,7 +33,7 @@ namespace NKikimr {
             LOG_DEBUG(ctx, BS_VDISK_GC,
                     VDISKP(HullCtx->VCtx->VDiskLogPrefix,
                             "TEvVGetBarrierResult: %s", Result->ToString().data()));
-            SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie);
+            SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
             ctx.Send(ParentId, new TEvents::TEvActorDied);
             Die(ctx);
         }

ydb/core/blobstorage/vdisk/query/query_base.h

@@ -118,7 +118,7 @@ namespace NKikimr {
                 ctx.Send(ReplSchedulerId, new TEvBlobStorage::TEvEnrichNotYet(BatcherCtx->OrigEv, std::move(Result)));
             } else {
                 // send reply event to sender
-                SendVDiskResponse(ctx, BatcherCtx->OrigEv->Sender, Result.release(), BatcherCtx->OrigEv->Cookie);
+                SendVDiskResponse(ctx, BatcherCtx->OrigEv->Sender, Result.release(), BatcherCtx->OrigEv->Cookie, QueryCtx->HullCtx->VCtx->VDiskLogPrefix, QueryCtx->HullCtx->VCtx->OOSMonGroup);
             }
 
             ctx.Send(ParentId, new TEvents::TEvActorDied);

ydb/core/blobstorage/vdisk/query/query_dumpdb.h

@@ -47,7 +47,7 @@ namespace NKikimr {
 
             // send result
             Result->SetResult(str.Str());
-            SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie);
+            SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
             TThis::Die(ctx);
         }
 

ydb/core/blobstorage/vdisk/query/query_public.cpp

@@ -158,7 +158,7 @@ namespace NKikimr {
         LOG_DEBUG(ctx, NKikimrServices::BS_VDISK_OTHER,
                 VDISKP(vctx->VDiskLogPrefix,
                     "TEvVDbStatResult: %s", result->ToString().data()));
-        SendVDiskResponse(ctx, ev->Sender, result.release(), ev->Cookie);
+        SendVDiskResponse(ctx, ev->Sender, result.release(), ev->Cookie, vctx->VDiskLogPrefix, vctx->OOSMonGroup);
     }
 
     template <class TKey, class TMemRec>

ydb/core/blobstorage/vdisk/query/query_statdb.h

@@ -29,10 +29,10 @@ namespace NKikimr {
                 const bool prettyPrint = Ev->Get()->Record.GetPrettyPrint();
                 CalculateStat(str, prettyPrint);
                 Result->SetResult(str.Str());
-                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie);
+                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
             } else {
                 CalculateStat(Result);
-                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie);
+                SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
             }
             ctx.Send(ParentId, new TEvents::TEvActorDied);
             TThis::Die(ctx);

ydb/core/blobstorage/vdisk/query/query_stathuge.cpp

@@ -22,7 +22,7 @@ namespace NKikimr {
             CalculateUsedHugeChunks(str, prettyPrint);
 
             Result->SetResult(str.Str());
-            SendVDiskResponse(ctx, Ev->Sender, Result.release(), 0);
+            SendVDiskResponse(ctx, Ev->Sender, Result.release(), 0, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
             ctx.Send(ParentId, new TEvents::TEvActorDied);
             TThis::Die(ctx);
         }

ydb/core/blobstorage/vdisk/query/query_stattablet.cpp

@@ -24,7 +24,7 @@ namespace NKikimr {
             ProcessLogoBlobs(str, tabletId, prettyPrint);
 
             Result->SetResult(str.Str());
-            SendVDiskResponse(ctx, Ev->Sender, Result.release(), 0);
+            SendVDiskResponse(ctx, Ev->Sender, Result.release(), 0, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
             ctx.Send(ParentId, new TEvents::TEvActorDied);
             TThis::Die(ctx);
         }

ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp

@@ -554,7 +554,7 @@ namespace NKikimr {
         std::set<TActorId> DonorQueryActors;
 
         void Handle(TEvBlobStorage::TEvEnrichNotYet::TPtr ev) {
-            DonorQueryActors.insert(Register(new TDonorQueryActor(*ev->Get(), Donors)));
+            DonorQueryActors.insert(Register(new TDonorQueryActor(*ev->Get(), Donors, ReplCtx->VCtx->VDiskLogPrefix, ReplCtx->VCtx->OOSMonGroup)));
         }
 
         void Handle(TEvents::TEvActorDied::TPtr ev) {

ydb/core/blobstorage/vdisk/repl/query_donor.h

@@ -12,14 +12,19 @@ namespace NKikimr {
         TActorId ParentId;
         std::deque<std::pair<TVDiskID, TActorId>> Donors;
         TDynBitMap UnresolvedItems;
+        TString VDiskLogPrefix;
+        std::shared_ptr<NMonGroup::TOutOfSpaceGroup> OOSMonGroup;
 
     public:
-        TDonorQueryActor(TEvBlobStorage::TEvEnrichNotYet& msg, std::deque<std::pair<TVDiskID, TActorId>> donors)
+        TDonorQueryActor(TEvBlobStorage::TEvEnrichNotYet& msg, std::deque<std::pair<TVDiskID, TActorId>> donors,
+                        const TString& vDiskLogPrefix, std::shared_ptr<NMonGroup::TOutOfSpaceGroup> monGroup)
             : Query(msg.Query->Release().Release())
             , Sender(msg.Query->Sender)
             , Cookie(msg.Query->Cookie)
             , Result(std::move(msg.Result))
             , Donors(std::move(donors))
+            , VDiskLogPrefix(vDiskLogPrefix)
+            , OOSMonGroup(std::move(monGroup))
         {
             Y_ABORT_UNLESS(!Query->Record.HasRangeQuery());
         }
@@ -108,7 +113,7 @@ namespace NKikimr {
         void PassAway() override {
             LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_VDISK_GET, SelfId() << " finished query");
             Send(ParentId, new TEvents::TEvActorDied);
-            SendVDiskResponse(TActivationContext::AsActorContext(), Sender, Result.release(), Cookie);
+            SendVDiskResponse(TActivationContext::AsActorContext(), Sender, Result.release(), Cookie, VDiskLogPrefix, OOSMonGroup);
             TActorBootstrapped::PassAway();
         }