Skip to content

Commit 9b24da3

Browse files
alexd65536alexd65536
authored
make throttling parameters configurable via CMS (#12911)
1 parent cf344b6 commit 9b24da3

File tree

9 files changed

+79
-50
lines changed

9 files changed

+79
-50
lines changed

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ TNodeWarden::TNodeWarden(const TIntrusivePtr<TNodeWardenConfig> &cfg)
3030
, MaxSyncLogChunksInFlightSSD(10, 1, 1024)
3131
, DefaultHugeGarbagePerMille(300, 1, 1000)
3232
, HugeDefragFreeSpaceBorderPerMille(260, 1, 1000)
33+
, ThrottlingDeviceSpeed(50 << 20, 1 << 20, 10ull << 30)
34+
, ThrottlingMinSstCount(100, 1, 1000)
35+
, ThrottlingMaxSstCount(250, 1, 1000)
36+
, ThrottlingMinInplacedSize(20ull << 30, 1 << 20, 500ull < 30)
37+
, ThrottlingMaxInplacedSize(60ull << 30, 1 << 20, 500ull < 30)
3338
, MaxCommonLogChunksHDD(200, 1, 1'000'000)
3439
, MaxCommonLogChunksSSD(200, 1, 1'000'000)
3540
, CostMetricsParametersByMedia({
@@ -336,6 +341,13 @@ void TNodeWarden::Bootstrap() {
336341
icb->RegisterSharedControl(MaxSyncLogChunksInFlightSSD, "VDiskControls.MaxSyncLogChunksInFlightSSD");
337342
icb->RegisterSharedControl(DefaultHugeGarbagePerMille, "VDiskControls.DefaultHugeGarbagePerMille");
338343
icb->RegisterSharedControl(HugeDefragFreeSpaceBorderPerMille, "VDiskControls.HugeDefragFreeSpaceBorderPerMille");
344+
345+
icb->RegisterSharedControl(ThrottlingDeviceSpeed, "VDiskControls.ThrottlingDeviceSpeed");
346+
icb->RegisterSharedControl(ThrottlingMinSstCount, "VDiskControls.ThrottlingMinSstCount");
347+
icb->RegisterSharedControl(ThrottlingMaxSstCount, "VDiskControls.ThrottlingMaxSstCount");
348+
icb->RegisterSharedControl(ThrottlingMinInplacedSize, "VDiskControls.ThrottlingMinInplacedSize");
349+
icb->RegisterSharedControl(ThrottlingMaxInplacedSize, "VDiskControls.ThrottlingMaxInplacedSize");
350+
339351
icb->RegisterSharedControl(MaxCommonLogChunksHDD, "PDiskControls.MaxCommonLogChunksHDD");
340352
icb->RegisterSharedControl(MaxCommonLogChunksSSD, "PDiskControls.MaxCommonLogChunksSSD");
341353

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,13 @@ namespace NKikimr::NStorage {
168168
TControlWrapper MaxSyncLogChunksInFlightSSD;
169169
TControlWrapper DefaultHugeGarbagePerMille;
170170
TControlWrapper HugeDefragFreeSpaceBorderPerMille;
171+
172+
TControlWrapper ThrottlingDeviceSpeed;
173+
TControlWrapper ThrottlingMinSstCount;
174+
TControlWrapper ThrottlingMaxSstCount;
175+
TControlWrapper ThrottlingMinInplacedSize;
176+
TControlWrapper ThrottlingMaxInplacedSize;
177+
171178
TControlWrapper MaxCommonLogChunksHDD;
172179
TControlWrapper MaxCommonLogChunksSSD;
173180

‎ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,12 @@ namespace NKikimr::NStorage {
199199
vdiskConfig->MaxSyncLogChunksInFlight = MaxSyncLogChunksInFlightSSD;
200200
}
201201

202+
vdiskConfig->ThrottlingDeviceSpeed = ThrottlingDeviceSpeed;
203+
vdiskConfig->ThrottlingMinSstCount = ThrottlingMinSstCount;
204+
vdiskConfig->ThrottlingMaxSstCount = ThrottlingMaxSstCount;
205+
vdiskConfig->ThrottlingMinInplacedSize = ThrottlingMinInplacedSize;
206+
vdiskConfig->ThrottlingMaxInplacedSize = ThrottlingMaxInplacedSize;
207+
202208
vdiskConfig->CostMetricsParametersByMedia = CostMetricsParametersByMedia;
203209

204210
vdiskConfig->FeatureFlags = Cfg->FeatureFlags;

‎ydb/core/blobstorage/vdisk/common/vdisk_config.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,13 @@ namespace NKikimr {
242242
bool UseCostTracker = true;
243243
TCostMetricsParametersByMedia CostMetricsParametersByMedia;
244244

245+
///////////// THROTTLING SETTINGS //////////////////
246+
TControlWrapper ThrottlingDeviceSpeed;
247+
TControlWrapper ThrottlingMinSstCount;
248+
TControlWrapper ThrottlingMaxSstCount;
249+
TControlWrapper ThrottlingMinInplacedSize;
250+
TControlWrapper ThrottlingMaxInplacedSize;
251+
245252
///////////// FEATURE FLAGS ////////////////////////
246253
NKikimrConfig::TFeatureFlags FeatureFlags;
247254

‎ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1982,16 +1982,14 @@ namespace NKikimr {
19821982
this->PrivateHandle(ev, ctx);
19831983
};
19841984
NMonGroup::TSkeletonOverloadGroup overloadMonGroup(VCtx->VDiskCounters, "subsystem", "emergency");
1985-
OverloadHandler = std::make_unique<TOverloadHandler>(VCtx, PDiskCtx, Hull,
1985+
OverloadHandler = std::make_unique<TOverloadHandler>(Config, VCtx, PDiskCtx, Hull,
19861986
std::move(overloadMonGroup), std::move(vMovedPatch), std::move(vPatchStart), std::move(vput),
19871987
std::move(vMultiPutHandler), std::move(loc), std::move(aoput));
19881988
ScheduleWakeupEmergencyPutQueue(ctx);
19891989

19901990
// actualize weights before we start
19911991
OverloadHandler->ActualizeWeights(ctx, AllEHullDbTypes, true);
19921992

1993-
OverloadHandler->RegisterIcbControls(AppData()->Icb);
1994-
19951993
// run Anubis
19961994
if (Config->RunAnubis && !Config->BaseInfo.DonorMode) {
19971995
auto anubisCtx = std::make_shared<TAnubisCtx>(HullCtx, ctx.SelfID,

‎ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp

Lines changed: 17 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -180,31 +180,7 @@ namespace NKikimr {
180180
class TThrottlingController {
181181
private:
182182
NMonGroup::TSkeletonOverloadGroup& Mon;
183-
184-
struct TControls {
185-
TControlWrapper DeviceSpeed;
186-
TControlWrapper MinSstCount;
187-
TControlWrapper MaxSstCount;
188-
TControlWrapper MinInplacedSize;
189-
TControlWrapper MaxInplacedSize;
190-
191-
TControls()
192-
: DeviceSpeed(50 << 20, 1 << 20, 1 << 30)
193-
, MinSstCount(100, 1, 1000)
194-
, MaxSstCount(250, 1, 1000)
195-
, MinInplacedSize(20ull << 30, 0, 500ull < 30)
196-
, MaxInplacedSize(60ull << 30, 0, 500ull < 30)
197-
{}
198-
199-
void Register(TIntrusivePtr<TControlBoard> icb) {
200-
icb->RegisterSharedControl(DeviceSpeed, "VDiskControls.ThrottlingDeviceSpeed");
201-
icb->RegisterSharedControl(MinSstCount, "VDiskControls.ThrottlingMinSstCount");
202-
icb->RegisterSharedControl(MaxSstCount, "VDiskControls.ThrottlingMaxSstCount");
203-
icb->RegisterSharedControl(MinInplacedSize, "VDiskControls.ThrottlingMinInplacedSize");
204-
icb->RegisterSharedControl(MaxInplacedSize, "VDiskControls.ThrottlingMaxInplacedSize");
205-
}
206-
};
207-
TControls Controls;
183+
TIntrusivePtr<TVDiskConfig> VCfg;
208184

209185
ui64 CurrentSstCount = 0;
210186
ui64 CurrentInplacedSize = 0;
@@ -228,17 +204,17 @@ namespace NKikimr {
228204
}
229205

230206
ui64 CalcSstCountSpeedLimit() const {
231-
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
232-
ui64 minSstCount = (ui64)Controls.MinSstCount;
233-
ui64 maxSstCount = (ui64)Controls.MaxSstCount;
207+
ui64 deviceSpeed = (ui64)VCfg->ThrottlingDeviceSpeed;
208+
ui64 minSstCount = (ui64)VCfg->ThrottlingMinSstCount;
209+
ui64 maxSstCount = (ui64)VCfg->ThrottlingMaxSstCount;
234210

235211
return LinearInterpolation(CurrentSstCount, minSstCount, maxSstCount, deviceSpeed);
236212
}
237213

238214
ui64 CalcInplacedSizeSpeedLimit() const {
239-
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
240-
ui64 minInplacedSize = (ui64)Controls.MinInplacedSize;
241-
ui64 maxInplacedSize = (ui64)Controls.MaxInplacedSize;
215+
ui64 deviceSpeed = (ui64)VCfg->ThrottlingDeviceSpeed;
216+
ui64 minInplacedSize = (ui64)VCfg->ThrottlingMinInplacedSize;
217+
ui64 maxInplacedSize = (ui64)VCfg->ThrottlingMaxInplacedSize;
242218

243219
return LinearInterpolation(CurrentInplacedSize, minInplacedSize, maxInplacedSize, deviceSpeed);
244220
}
@@ -250,17 +226,16 @@ namespace NKikimr {
250226
}
251227

252228
public:
253-
explicit TThrottlingController(NMonGroup::TSkeletonOverloadGroup& mon)
229+
explicit TThrottlingController(
230+
const TIntrusivePtr<TVDiskConfig> &vcfg,
231+
NMonGroup::TSkeletonOverloadGroup& mon)
254232
: Mon(mon)
233+
, VCfg(vcfg)
255234
{}
256235

257-
void RegisterIcbControls(TIntrusivePtr<TControlBoard> icb) {
258-
Controls.Register(icb);
259-
}
260-
261236
bool IsActive() const {
262-
ui64 minSstCount = (ui64)Controls.MinSstCount;
263-
ui64 minInplacedSize = (ui64)Controls.MinInplacedSize;
237+
ui64 minSstCount = (ui64)VCfg->ThrottlingMinSstCount;
238+
ui64 minInplacedSize = (ui64)VCfg->ThrottlingMinInplacedSize;
264239

265240
return CurrentSstCount > minSstCount ||
266241
CurrentInplacedSize > minInplacedSize;
@@ -297,7 +272,7 @@ namespace NKikimr {
297272
if (!IsActive()) {
298273
CurrentTime = {};
299274
AvailableBytes = 0;
300-
CurrentSpeedLimit = (ui64)Controls.DeviceSpeed;
275+
CurrentSpeedLimit = (ui64)VCfg->ThrottlingDeviceSpeed;
301276
} else {
302277
if (!prevActive) {
303278
CurrentTime = now;
@@ -315,7 +290,7 @@ namespace NKikimr {
315290
}
316291
auto us = (now - CurrentTime).MicroSeconds();
317292
AvailableBytes += CurrentSpeedLimit * us / 1000000;
318-
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
293+
ui64 deviceSpeed = (ui64)VCfg->ThrottlingDeviceSpeed;
319294
AvailableBytes = Min(AvailableBytes, deviceSpeed);
320295
CurrentTime = now;
321296
}
@@ -325,6 +300,7 @@ namespace NKikimr {
325300
// TOverloadHandler
326301
///////////////////////////////////////////////////////////////////////////////////////////////////
327302
TOverloadHandler::TOverloadHandler(
303+
const TIntrusivePtr<TVDiskConfig> &vcfg,
328304
const TIntrusivePtr<TVDiskContext> &vctx,
329305
const TPDiskCtxPtr &pdiskCtx,
330306
std::shared_ptr<THull> hull,
@@ -340,7 +316,7 @@ namespace NKikimr {
340316
, EmergencyQueue(new TEmergencyQueue(Mon, std::move(vMovedPatch), std::move(vPatchStart), std::move(vput),
341317
std::move(vMultiPut), std::move(loc), std::move(aoput)))
342318
, DynamicPDiskWeightsManager(std::make_shared<TDynamicPDiskWeightsManager>(vctx, pdiskCtx))
343-
, ThrottlingController(new TThrottlingController(Mon))
319+
, ThrottlingController(new TThrottlingController(vcfg, Mon))
344320
{}
345321

346322
TOverloadHandler::~TOverloadHandler() {}
@@ -462,10 +438,6 @@ namespace NKikimr {
462438
KickInFlight = false;
463439
}
464440

465-
void TOverloadHandler::RegisterIcbControls(TIntrusivePtr<TControlBoard> icb) {
466-
ThrottlingController->RegisterIcbControls(icb);
467-
}
468-
469441
template <class TEv>
470442
inline bool TOverloadHandler::PostponeEvent(TAutoPtr<TEventHandle<TEv>> &ev) {
471443
if (DynamicPDiskWeightsManager->StopPuts() ||

‎ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ namespace NKikimr {
6161
class TOverloadHandler {
6262
public:
6363
TOverloadHandler(
64+
const TIntrusivePtr<TVDiskConfig> &vcfg,
6465
const TIntrusivePtr<TVDiskContext> &vctx,
6566
const TPDiskCtxPtr &pdiskCtx,
6667
std::shared_ptr<THull> hull,
@@ -91,8 +92,6 @@ namespace NKikimr {
9192

9293
void OnKickEmergencyPutQueue();
9394

94-
void RegisterIcbControls(TIntrusivePtr<TControlBoard> icb);
95-
9695
private:
9796
std::shared_ptr<THull> Hull;
9897
NMonGroup::TSkeletonOverloadGroup Mon;

‎ydb/core/cms/json_proxy_proto.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ class TJsonProxyProto : public TActorBootstrapped<TJsonProxyProto> {
8686
return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TDSProxyControls::descriptor(), ctx);
8787
else if (name == ".NKikimrConfig.TImmediateControlsConfig.TBlobStorageControllerControls")
8888
return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TBlobStorageControllerControls::descriptor(), ctx);
89+
else if (name == ".NKikimrConfig.TImmediateControlsConfig.TTableServiceControls")
90+
return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TTableServiceControls::descriptor(), ctx);
8991
}
9092

9193
ctx.Send(RequestEvent->Sender,

‎ydb/core/protos/config.proto

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,6 +1375,32 @@ message TImmediateControlsConfig {
13751375
MinValue: 1,
13761376
MaxValue: 1000,
13771377
DefaultValue: 260 }];
1378+
1379+
optional uint64 ThrottlingDeviceSpeed = 14 [(ControlOptions) = {
1380+
Description: "Maximum allowed speed when throttling is active",
1381+
MinValue: 1048576,
1382+
MaxValue: 10737418240,
1383+
DefaultValue: 52428800 }];
1384+
optional uint64 ThrottlingMinSstCount = 15 [(ControlOptions) = {
1385+
Description: "Minimum level 0 SST count - throttling is turned on",
1386+
MinValue: 1,
1387+
MaxValue: 1000,
1388+
DefaultValue: 100 }];
1389+
optional uint64 ThrottlingMaxSstCount = 16 [(ControlOptions) = {
1390+
Description: "Maximum level 0 SST count - throttling speed is zero",
1391+
MinValue: 1,
1392+
MaxValue: 1000,
1393+
DefaultValue: 250 }];
1394+
optional uint64 ThrottlingMinInplacedSize = 17 [(ControlOptions) = {
1395+
Description: "Minimum size of all inplaced blobs - throttling is turned on",
1396+
MinValue: 1048576,
1397+
MaxValue: 536870912000,
1398+
DefaultValue: 21474836480 }];
1399+
optional uint64 ThrottlingMaxInplacedSize = 18 [(ControlOptions) = {
1400+
Description: "Maximum size of all inplaced blobs - throttling speed is zero",
1401+
MinValue: 1048576,
1402+
MaxValue: 536870912000,
1403+
DefaultValue: 64424509440 }];
13781404
}
13791405

13801406
message TTabletControls {

0 commit comments

Comments
 (0)