Skip to content

Commit 15f4ea5

Browse files
robdrynkinva-kuznecov
robdrynkin
and
va-kuznecov
authored
Add logging and monitoring for OOS in vdisk (#2842) (#3036)
Co-authored-by: Vlad Kuznetsov <va.kuznecov@physics.msu.ru>
1 parent 941b959 commit 15f4ea5

31 files changed

+182
-79
lines changed

‎ydb/core/blobstorage/vdisk/common/blobstorage_status.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ namespace NKikimr {
3636
SetRacingGroupInfo(record, Result->Record, GroupInfo);
3737
LOG_DEBUG(ctx, BS_VDISK_OTHER, VDISKP(VCtx->VDiskLogPrefix, "TEvVStatusResult Request# {%s} Response# {%s}",
3838
SingleLineProto(record).data(), SingleLineProto(Result->Record).data()));
39-
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel());
39+
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel(), VCtx->VDiskLogPrefix, VCtx->OOSMonGroup);
4040
Die(ctx);
4141
return;
4242
}
@@ -72,7 +72,7 @@ namespace NKikimr {
7272
ctx.Send(NotifyId, new TEvents::TEvActorDied());
7373
LOG_DEBUG(ctx, BS_VDISK_GET,
7474
VDISKP(VCtx->VDiskLogPrefix, "TEvVStatusResult"));
75-
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel());
75+
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, Ev->GetChannel(), VCtx->VDiskLogPrefix, VCtx->OOSMonGroup);
7676
Die(ctx);
7777
}
7878
}

‎ydb/core/blobstorage/vdisk/common/vdisk_context.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ namespace NKikimr {
3838
, VDiskMemCounters(vdiskCounters->GetSubgroup("subsystem", "memhull"))
3939
, Histograms(VDiskCounters, type)
4040
, IFaceMonGroup(std::make_shared<NMonGroup::TVDiskIFaceGroup>(VDiskCounters, "subsystem", "interface"))
41+
, OOSMonGroup(std::make_shared<NMonGroup::TOutOfSpaceGroup>(VDiskCounters, "subsystem", "oos"))
4142
, GroupId(selfVDisk.GroupID)
4243
, ShortSelfVDisk(selfVDisk)
4344
, VDiskLogPrefix(GenerateVDiskLogPrefix(selfVDisk, donorMode))

‎ydb/core/blobstorage/vdisk/common/vdisk_context.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ namespace NKikimr {
4242
// latency histograms
4343
NVDiskMon::THistograms Histograms;
4444
std::shared_ptr<NMonGroup::TVDiskIFaceGroup> IFaceMonGroup;
45+
std::shared_ptr<NMonGroup::TOutOfSpaceGroup> OOSMonGroup;
4546
// Self VDisk related info
4647
const ui32 GroupId;
4748
const TVDiskIdShort ShortSelfVDisk;

‎ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,29 @@ public:
561561
}
562562
};
563563

564+
///////////////////////////////////////////////////////////////////////////////////
565+
// TOutOfSpaceGroup
566+
///////////////////////////////////////////////////////////////////////////////////
567+
class TOutOfSpaceGroup : public TBase {
568+
public:
569+
GROUP_CONSTRUCTOR(TOutOfSpaceGroup)
570+
{
571+
COUNTER_INIT(ResponsesWithDiskSpaceRed, true);
572+
COUNTER_INIT(ResponsesWithDiskSpaceOrange, true);
573+
COUNTER_INIT(ResponsesWithDiskSpacePreOrange, true);
574+
COUNTER_INIT(ResponsesWithDiskSpaceLightOrange, true);
575+
COUNTER_INIT(ResponsesWithDiskSpaceYellowStop, true);
576+
COUNTER_INIT(ResponsesWithDiskSpaceLightYellowMove, true);
577+
}
578+
579+
COUNTER_DEF(ResponsesWithDiskSpaceRed);
580+
COUNTER_DEF(ResponsesWithDiskSpaceOrange);
581+
COUNTER_DEF(ResponsesWithDiskSpacePreOrange);
582+
COUNTER_DEF(ResponsesWithDiskSpaceLightOrange);
583+
COUNTER_DEF(ResponsesWithDiskSpaceYellowStop);
584+
COUNTER_DEF(ResponsesWithDiskSpaceLightYellowMove);
585+
};
586+
564587
} // NMonGroup
565588
} // NKikimr
566589

‎ydb/core/blobstorage/vdisk/common/vdisk_response.cpp‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,40 @@
11
#include "vdisk_response.h"
22
#include "vdisk_events.h"
33
#include <ydb/core/base/interconnect_channels.h>
4+
#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_space_color.h>
45
#include <util/system/datetime.h>
56

67
namespace NKikimr {
78

8-
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie) {
9+
void LogOOSStatus(ui32 flags, const TLogoBlobID& blobId, const TString& vDiskLogPrefix);
10+
void UpdateMonOOSStatus(ui32 flags, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup);
11+
12+
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup) {
913
ui32 channel = TInterconnectChannels::IC_BLOBSTORAGE;
1014
if (TEvVResultBase *base = dynamic_cast<TEvVResultBase *>(ev)) {
1115
channel = base->GetChannelToSend();
1216
}
13-
SendVDiskResponse(ctx, recipient, ev, cookie, channel);
17+
SendVDiskResponse(ctx, recipient, ev, cookie, channel, vDiskLogPrefix, monGroup);
1418
}
1519

16-
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel) {
20+
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup) {
21+
switch(ev->Type()) {
22+
case TEvBlobStorage::TEvVPutResult::EventType: {
23+
TEvBlobStorage::TEvVPutResult* event = static_cast<TEvBlobStorage::TEvVPutResult *>(ev);
24+
LogOOSStatus(event->Record.GetStatusFlags(), LogoBlobIDFromLogoBlobID(event->Record.GetBlobID()), vDiskLogPrefix);
25+
UpdateMonOOSStatus(event->Record.GetStatusFlags(), monGroup);
26+
break;
27+
}
28+
case TEvBlobStorage::TEvVMultiPutResult::EventType: {
29+
TEvBlobStorage::TEvVMultiPutResult *event = static_cast<TEvBlobStorage::TEvVMultiPutResult *>(ev);
30+
for (ui64 i = 0; i < event->Record.ItemsSize(); ++i) {
31+
const auto& item = event->Record.GetItems(i);
32+
LogOOSStatus(item.GetStatusFlags(), LogoBlobIDFromLogoBlobID(item.GetBlobID()), vDiskLogPrefix);
33+
UpdateMonOOSStatus(item.GetStatusFlags(), monGroup);
34+
}
35+
}
36+
}
37+
1738
switch (const ui32 type = ev->Type()) {
1839
#define HANDLE_EVENT(T) \
1940
case TEvBlobStorage::T::EventType: { \
@@ -45,4 +66,34 @@ void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEve
4566
}
4667
}
4768

69+
void LogOOSStatus(ui32 flags, const TLogoBlobID& blobId, const TString& vDiskLogPrefix) {
70+
if (!TlsActivationContext) {
71+
return;
72+
}
73+
74+
LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_VDISK_CHUNKS,
75+
vDiskLogPrefix << "Disk space status changed to " <<
76+
TPDiskSpaceColor_Name(StatusFlagToSpaceColor(flags)) << " on blob " << blobId.ToString());
77+
}
78+
79+
void UpdateMonOOSStatus(ui32 flags, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup) {
80+
if (!monGroup) {
81+
return;
82+
}
83+
84+
if (flags & NKikimrBlobStorage::StatusDiskSpaceRed) {
85+
monGroup->ResponsesWithDiskSpaceRed().Inc();
86+
} else if (flags & NKikimrBlobStorage::StatusDiskSpaceOrange) {
87+
monGroup->ResponsesWithDiskSpaceOrange().Inc();
88+
} else if (flags & NKikimrBlobStorage::StatusDiskSpaceLightOrange) {
89+
monGroup->ResponsesWithDiskSpaceLightOrange().Inc();
90+
} else if (flags & NKikimrBlobStorage::StatusDiskSpacePreOrange) {
91+
monGroup->ResponsesWithDiskSpacePreOrange().Inc();
92+
} else if (flags & NKikimrBlobStorage::StatusDiskSpaceYellowStop) {
93+
monGroup->ResponsesWithDiskSpaceYellowStop().Inc();
94+
} else if (flags & NKikimrBlobStorage::StatusDiskSpaceLightYellowMove) {
95+
monGroup->ResponsesWithDiskSpaceLightYellowMove().Inc();
96+
}
97+
}
98+
4899
}//NKikimr

‎ydb/core/blobstorage/vdisk/common/vdisk_response.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
#pragma once
22
#include "defs.h"
3+
#include "vdisk_mongroups.h"
34

45
namespace NKikimr {
56

6-
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie);
7+
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup);
78

8-
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel);
9+
void SendVDiskResponse(const TActorContext &ctx, const TActorId &recipient, IEventBase *ev, ui64 cookie, ui32 channel, const TString& vDiskLogPrefix, const std::shared_ptr<NMonGroup::TOutOfSpaceGroup>& monGroup);
910

1011
}//NKikimr

‎ydb/core/blobstorage/vdisk/query/query_barrier.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ namespace NKikimr {
3333
LOG_DEBUG(ctx, BS_VDISK_GC,
3434
VDISKP(HullCtx->VCtx->VDiskLogPrefix,
3535
"TEvVGetBarrierResult: %s", Result->ToString().data()));
36-
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie);
36+
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
3737
ctx.Send(ParentId, new TEvents::TEvActorDied);
3838
Die(ctx);
3939
}

‎ydb/core/blobstorage/vdisk/query/query_base.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ namespace NKikimr {
118118
ctx.Send(ReplSchedulerId, new TEvBlobStorage::TEvEnrichNotYet(BatcherCtx->OrigEv, std::move(Result)));
119119
} else {
120120
// send reply event to sender
121-
SendVDiskResponse(ctx, BatcherCtx->OrigEv->Sender, Result.release(), BatcherCtx->OrigEv->Cookie);
121+
SendVDiskResponse(ctx, BatcherCtx->OrigEv->Sender, Result.release(), BatcherCtx->OrigEv->Cookie, QueryCtx->HullCtx->VCtx->VDiskLogPrefix, QueryCtx->HullCtx->VCtx->OOSMonGroup);
122122
}
123123

124124
ctx.Send(ParentId, new TEvents::TEvActorDied);

‎ydb/core/blobstorage/vdisk/query/query_dumpdb.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ namespace NKikimr {
4747

4848
// send result
4949
Result->SetResult(str.Str());
50-
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie);
50+
SendVDiskResponse(ctx, Ev->Sender, Result.release(), Ev->Cookie, HullCtx->VCtx->VDiskLogPrefix, HullCtx->VCtx->OOSMonGroup);
5151
TThis::Die(ctx);
5252
}
5353

‎ydb/core/blobstorage/vdisk/query/query_public.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ namespace NKikimr {
158158
LOG_DEBUG(ctx, NKikimrServices::BS_VDISK_OTHER,
159159
VDISKP(vctx->VDiskLogPrefix,
160160
"TEvVDbStatResult: %s", result->ToString().data()));
161-
SendVDiskResponse(ctx, ev->Sender, result.release(), ev->Cookie);
161+
SendVDiskResponse(ctx, ev->Sender, result.release(), ev->Cookie, vctx->VDiskLogPrefix, vctx->OOSMonGroup);
162162
}
163163

164164
template <class TKey, class TMemRec>

0 commit comments

Comments
 (0)