Skip to content

Commit dd6b961

Browse files
alexd65536alexd65536
authored
add vdisk throttling according to disk space occupancy (#13098)
1 parent e3c91f5 commit dd6b961

File tree

8 files changed

+49
-8
lines changed

8 files changed

+49
-8
lines changed

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ TNodeWarden::TNodeWarden(const TIntrusivePtr<TNodeWardenConfig> &cfg)
3535
, ThrottlingMaxSstCount(250, 1, 1000)
3636
, ThrottlingMinInplacedSize(20ull << 30, 1 << 20, 500ull < 30)
3737
, ThrottlingMaxInplacedSize(60ull << 30, 1 << 20, 500ull < 30)
38+
, ThrottlingMinOccupancyPerMille(900, 1, 1000)
39+
, ThrottlingMaxOccupancyPerMille(950, 1, 1000)
3840
, MaxCommonLogChunksHDD(200, 1, 1'000'000)
3941
, MaxCommonLogChunksSSD(200, 1, 1'000'000)
4042
, CostMetricsParametersByMedia({
@@ -347,6 +349,8 @@ void TNodeWarden::Bootstrap() {
347349
icb->RegisterSharedControl(ThrottlingMaxSstCount, "VDiskControls.ThrottlingMaxSstCount");
348350
icb->RegisterSharedControl(ThrottlingMinInplacedSize, "VDiskControls.ThrottlingMinInplacedSize");
349351
icb->RegisterSharedControl(ThrottlingMaxInplacedSize, "VDiskControls.ThrottlingMaxInplacedSize");
352+
icb->RegisterSharedControl(ThrottlingMinOccupancyPerMille, "VDiskControls.ThrottlingMinOccupancyPerMille");
353+
icb->RegisterSharedControl(ThrottlingMaxOccupancyPerMille, "VDiskControls.ThrottlingMaxOccupancyPerMille");
350354

351355
icb->RegisterSharedControl(MaxCommonLogChunksHDD, "PDiskControls.MaxCommonLogChunksHDD");
352356
icb->RegisterSharedControl(MaxCommonLogChunksSSD, "PDiskControls.MaxCommonLogChunksSSD");

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,8 @@ namespace NKikimr::NStorage {
174174
TControlWrapper ThrottlingMaxSstCount;
175175
TControlWrapper ThrottlingMinInplacedSize;
176176
TControlWrapper ThrottlingMaxInplacedSize;
177+
TControlWrapper ThrottlingMinOccupancyPerMille;
178+
TControlWrapper ThrottlingMaxOccupancyPerMille;
177179

178180
TControlWrapper MaxCommonLogChunksHDD;
179181
TControlWrapper MaxCommonLogChunksSSD;

‎ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,8 @@ namespace NKikimr::NStorage {
204204
vdiskConfig->ThrottlingMaxSstCount = ThrottlingMaxSstCount;
205205
vdiskConfig->ThrottlingMinInplacedSize = ThrottlingMinInplacedSize;
206206
vdiskConfig->ThrottlingMaxInplacedSize = ThrottlingMaxInplacedSize;
207+
vdiskConfig->ThrottlingMinOccupancyPerMille = ThrottlingMinOccupancyPerMille;
208+
vdiskConfig->ThrottlingMaxOccupancyPerMille = ThrottlingMaxOccupancyPerMille;
207209

208210
vdiskConfig->CostMetricsParametersByMedia = CostMetricsParametersByMedia;
209211

‎ydb/core/blobstorage/vdisk/common/vdisk_config.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,8 @@ namespace NKikimr {
248248
TControlWrapper ThrottlingMaxSstCount;
249249
TControlWrapper ThrottlingMinInplacedSize;
250250
TControlWrapper ThrottlingMaxInplacedSize;
251+
TControlWrapper ThrottlingMinOccupancyPerMille;
252+
TControlWrapper ThrottlingMaxOccupancyPerMille;
251253

252254
///////////// FEATURE FLAGS ////////////////////////
253255
NKikimrConfig::TFeatureFlags FeatureFlags;

‎ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ public:
140140
COUNTER_INIT_IF_EXTENDED(ThrottlingIsActive, false);
141141
COUNTER_INIT_IF_EXTENDED(ThrottlingLevel0SstCount, false);
142142
COUNTER_INIT_IF_EXTENDED(ThrottlingAllLevelsInplacedSize, false);
143+
COUNTER_INIT_IF_EXTENDED(ThrottlingOccupancyPerMille, false);
143144
}
144145

145146
COUNTER_DEF(EmergencyMovedPatchQueueItems);
@@ -163,6 +164,7 @@ public:
163164
COUNTER_DEF(ThrottlingIsActive);
164165
COUNTER_DEF(ThrottlingLevel0SstCount);
165166
COUNTER_DEF(ThrottlingAllLevelsInplacedSize);
167+
COUNTER_DEF(ThrottlingOccupancyPerMille);
166168
};
167169

168170
///////////////////////////////////////////////////////////////////////////////////

‎ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ namespace NKikimr {
184184

185185
ui64 CurrentSstCount = 0;
186186
ui64 CurrentInplacedSize = 0;
187+
ui64 CurrentOccupancy = 0;
187188

188189
TInstant CurrentTime;
189190
ui64 CurrentSpeedLimit = 0;
@@ -219,10 +220,20 @@ namespace NKikimr {
219220
return LinearInterpolation(CurrentInplacedSize, minInplacedSize, maxInplacedSize, deviceSpeed);
220221
}
221222

223+
ui64 CalcOccupancySpeedLimit() const {
224+
ui64 deviceSpeed = (ui64)VCfg->ThrottlingDeviceSpeed;
225+
ui64 minOccupancy = (ui64)VCfg->ThrottlingMinOccupancyPerMille * 1000;
226+
ui64 maxOccupancy = (ui64)VCfg->ThrottlingMaxOccupancyPerMille * 1000;
227+
228+
return LinearInterpolation(CurrentOccupancy, minOccupancy, maxOccupancy, deviceSpeed);
229+
}
230+
222231
ui64 CalcCurrentSpeedLimit() const {
223232
ui64 sstCountSpeedLimit = CalcSstCountSpeedLimit();
224233
ui64 inplacedSizeSpeedLimit = CalcInplacedSizeSpeedLimit();
225-
return std::min(sstCountSpeedLimit, inplacedSizeSpeedLimit);
234+
ui64 occupancySpeedLimit = CalcOccupancySpeedLimit();
235+
236+
return std::min(occupancySpeedLimit, std::min(sstCountSpeedLimit, inplacedSizeSpeedLimit));
226237
}
227238

228239
public:
@@ -236,9 +247,11 @@ namespace NKikimr {
236247
bool IsActive() const {
237248
ui64 minSstCount = (ui64)VCfg->ThrottlingMinSstCount;
238249
ui64 minInplacedSize = (ui64)VCfg->ThrottlingMinInplacedSize;
250+
ui64 minOccupancy = (ui64)VCfg->ThrottlingMinOccupancyPerMille * 1000;
239251

240252
return CurrentSstCount > minSstCount ||
241-
CurrentInplacedSize > minInplacedSize;
253+
CurrentInplacedSize > minInplacedSize ||
254+
CurrentOccupancy > minOccupancy;
242255
}
243256

244257
TDuration BytesToDuration(ui64 bytes) const {
@@ -258,7 +271,7 @@ namespace NKikimr {
258271
return AvailableBytes;
259272
}
260273

261-
void UpdateState(TInstant now, ui64 sstCount, ui64 inplacedSize) {
274+
void UpdateState(TInstant now, ui64 sstCount, ui64 inplacedSize, float occupancy) {
262275
bool prevActive = IsActive();
263276

264277
CurrentSstCount = sstCount;
@@ -267,6 +280,9 @@ namespace NKikimr {
267280
CurrentInplacedSize = inplacedSize;
268281
Mon.ThrottlingAllLevelsInplacedSize() = inplacedSize;
269282

283+
CurrentOccupancy = occupancy * 1'000'000;
284+
Mon.ThrottlingOccupancyPerMille() = occupancy * 1000;
285+
270286
Mon.ThrottlingIsActive() = (ui64)IsActive();
271287

272288
if (!IsActive()) {
@@ -291,7 +307,7 @@ namespace NKikimr {
291307
auto us = (now - CurrentTime).MicroSeconds();
292308
AvailableBytes += CurrentSpeedLimit * us / 1000000;
293309
ui64 deviceSpeed = (ui64)VCfg->ThrottlingDeviceSpeed;
294-
AvailableBytes = Min(AvailableBytes, deviceSpeed);
310+
AvailableBytes = std::min(AvailableBytes, deviceSpeed);
295311
CurrentTime = now;
296312
}
297313
};
@@ -311,7 +327,8 @@ namespace NKikimr {
311327
TVMultiPutHandler &&vMultiPut,
312328
TLocalSyncDataHandler &&loc,
313329
TAnubisOsirisPutHandler &&aoput)
314-
: Hull(std::move(hull))
330+
: VCtx(vctx)
331+
, Hull(std::move(hull))
315332
, Mon(std::move(mon))
316333
, EmergencyQueue(new TEmergencyQueue(Mon, std::move(vMovedPatch), std::move(vPatchStart), std::move(vput),
317334
std::move(vMultiPut), std::move(loc), std::move(aoput)))
@@ -358,11 +375,12 @@ namespace NKikimr {
358375
auto& logoBlobsSnap = snapshot.LogoBlobsSnap; // TLogoBlobsSnapshot
359376
auto& sliceSnap = logoBlobsSnap.SliceSnap; // TLevelSliceSnapshot
360377

361-
auto sstCount = sliceSnap.GetLevel0SstsNum();
362-
auto dataInplacedSize = logoBlobsSnap.AllLevelsDataInplaced;
378+
ui64 sstCount = sliceSnap.GetLevel0SstsNum();
379+
ui64 dataInplacedSize = logoBlobsSnap.AllLevelsDataInplaced;
380+
float occupancy = 1.f - VCtx->GetOutOfSpaceState().GetFreeSpaceShare();
363381

364382
auto now = ctx.Now();
365-
ThrottlingController->UpdateState(now, sstCount, dataInplacedSize);
383+
ThrottlingController->UpdateState(now, sstCount, dataInplacedSize, occupancy);
366384

367385
if (ThrottlingController->IsActive()) {
368386
ThrottlingController->UpdateTime(now);

‎ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ namespace NKikimr {
9393
void OnKickEmergencyPutQueue();
9494

9595
private:
96+
TIntrusivePtr<TVDiskContext> VCtx;
9697
std::shared_ptr<THull> Hull;
9798
NMonGroup::TSkeletonOverloadGroup Mon;
9899
std::unique_ptr<TEmergencyQueue> EmergencyQueue;

‎ydb/core/protos/config.proto

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1399,6 +1399,16 @@ message TImmediateControlsConfig {
13991399
MinValue: 1048576,
14001400
MaxValue: 536870912000,
14011401
DefaultValue: 64424509440 }];
1402+
optional uint64 ThrottlingMinOccupancyPerMille = 19 [(ControlOptions) = {
1403+
Description: "Minimum occupancy of disk per mille - throttling is turned on",
1404+
MinValue: 1,
1405+
MaxValue: 1000,
1406+
DefaultValue: 900 }];
1407+
optional uint64 ThrottlingMaxOccupancyPerMille = 20 [(ControlOptions) = {
1408+
Description: "Maximum occupancy of disk per mille - throttling speed is zero",
1409+
MinValue: 1,
1410+
MaxValue: 1000,
1411+
DefaultValue: 950 }];
14021412
}
14031413

14041414
message TTabletControls {

0 commit comments

Comments
 (0)