Skip to content

Commit 2b91145

Browse files
serbel324serbel324
authored
Separate acceleration ICB settings for HDD and SSD (#11590)
1 parent 7bb4c8d commit 2b91145

20 files changed

+225
-93
lines changed

‎ydb/core/blobstorage/common/defs.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#pragma once
2+
3+
#include <ydb/core/blobstorage/defs.h>
4+
#include <util/datetime/base.h>

‎ydb/core/blobstorage/common/immediate_control_defaults.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#include "immediate_control_defaults.h"
2+
3+
namespace NKikimr {
4+
5+
TControlWrapper SlowDiskThresholdDefaultControl =
6+
TControlWrapper(std::round(DefaultSlowDiskThreshold * 1000), 1, 1'000'000);
7+
8+
TControlWrapper PredictedDelayMultiplierDefaultControl =
9+
TControlWrapper(std::round(DefaultPredictedDelayMultiplier * 1000), 0, 1'000'000);
10+
11+
TControlWrapper MaxNumOfSlowDisksDefaultControl =
12+
TControlWrapper(DefaultMaxNumOfSlowDisks, 1, 2);
13+
14+
TControlWrapper LongRequestThresholdDefaultControl =
15+
TControlWrapper(DefaultLongRequestThreshold.MilliSeconds(), 1, 1'000'000);
16+
17+
} // namespace NKikimr

‎ydb/core/blobstorage/common/immediate_control_defaults.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#pragma once
2+
3+
#include "defs.h"
4+
#include <ydb/core/control/immediate_control_board_wrapper.h>
5+
6+
namespace NKikimr {
7+
8+
constexpr bool DefaultEnablePutBatching = true;
9+
constexpr bool DefaultEnableVPatch = false;
10+
11+
constexpr float DefaultSlowDiskThreshold = 2;
12+
constexpr float DefaultPredictedDelayMultiplier = 1;
13+
constexpr TDuration DefaultLongRequestThreshold = TDuration::Seconds(50);
14+
constexpr ui32 DefaultMaxNumOfSlowDisks = 2;
15+
16+
extern TControlWrapper SlowDiskThresholdDefaultControl;
17+
extern TControlWrapper PredictedDelayMultiplierDefaultControl;
18+
extern TControlWrapper MaxNumOfSlowDisksDefaultControl;
19+
extern TControlWrapper LongRequestThresholdDefaultControl;
20+
21+
}

‎ydb/core/blobstorage/common/ya.make‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
LIBRARY()
2+
3+
PEERDIR(
4+
ydb/core/base
5+
)
6+
7+
SRCS(
8+
immediate_control_defaults.cpp
9+
)
10+
11+
END()

‎ydb/core/blobstorage/dsproxy/dsproxy.h‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <ydb/core/blobstorage/base/blobstorage_events.h>
1515
#include <ydb/core/blobstorage/base/transparent.h>
1616
#include <ydb/core/blobstorage/backpressure/queue_backpressure_client.h>
17+
#include <ydb/core/blobstorage/common/immediate_control_defaults.h>
1718
#include <ydb/library/actors/core/interconnect.h>
1819
#include <ydb/library/actors/wilson/wilson_span.h>
1920
#include <ydb/core/base/appdata_fwd.h>
@@ -52,14 +53,6 @@ const ui32 MaxRequestSize = 1000;
5253

5354
const ui32 MaskSizeBits = 32;
5455

55-
constexpr bool DefaultEnablePutBatching = true;
56-
constexpr bool DefaultEnableVPatch = false;
57-
58-
constexpr double DefaultSlowDiskThreshold = 2;
59-
constexpr double DefaultPredictedDelayMultiplier = 1;
60-
constexpr TDuration DefaultLongRequestThreshold = TDuration::Seconds(50);
61-
constexpr ui32 DefaultMaxNumOfSlowDisks = 2;
62-
6356
constexpr bool WithMovingPatchRequestToStaticNode = true;
6457

6558
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -515,15 +508,30 @@ IActor* CreateBlobStorageGroupAssimilateRequest(TBlobStorageGroupAssimilateParam
515508

516509
IActor* CreateBlobStorageGroupEjectedProxy(ui32 groupId, TIntrusivePtr<TDsProxyNodeMon> &nodeMon);
517510

511+
struct TBlobStorageProxyControlWrappers {
512+
TMemorizableControlWrapper EnablePutBatching;
513+
TMemorizableControlWrapper EnableVPatch;
514+
515+
TMemorizableControlWrapper LongRequestThresholdMs = LongRequestThresholdDefaultControl;
516+
517+
#define DEVICE_TYPE_SEPECIFIC_MEMORIZABLE_CONTROLS(prefix) \
518+
TMemorizableControlWrapper prefix = prefix##DefaultControl; \
519+
TMemorizableControlWrapper prefix##HDD = prefix##DefaultControl; \
520+
TMemorizableControlWrapper prefix##SSD = prefix##DefaultControl
521+
522+
// Acceleration parameters
523+
DEVICE_TYPE_SEPECIFIC_MEMORIZABLE_CONTROLS(SlowDiskThreshold);
524+
DEVICE_TYPE_SEPECIFIC_MEMORIZABLE_CONTROLS(PredictedDelayMultiplier);
525+
DEVICE_TYPE_SEPECIFIC_MEMORIZABLE_CONTROLS(MaxNumOfSlowDisks);
526+
527+
#undef DEVICE_TYPE_SEPECIFIC_MEMORIZABLE_CONTROLS
528+
529+
};
530+
518531
struct TBlobStorageProxyParameters {
519532
bool UseActorSystemTimeInBSQueue = false;
520533

521-
const TControlWrapper& EnablePutBatching;
522-
const TControlWrapper& EnableVPatch;
523-
const TControlWrapper& SlowDiskThreshold;
524-
const TControlWrapper& PredictedDelayMultiplier;
525-
const TControlWrapper& LongRequestThresholdMs = TControlWrapper(DefaultLongRequestThreshold.MilliSeconds(), 1, 1'000'000);
526-
const TControlWrapper& MaxNumOfSlowDisks = TControlWrapper(DefaultMaxNumOfSlowDisks, 1, 2);
534+
TBlobStorageProxyControlWrappers Controls;
527535
};
528536

529537
IActor* CreateBlobStorageGroupProxyConfigured(TIntrusivePtr<TBlobStorageGroupInfo>&& info,

‎ydb/core/blobstorage/dsproxy/dsproxy_impl.cpp‎

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,7 @@ namespace NKikimr {
1515
, IsEjected(false)
1616
, ForceWaitAllDrives(forceWaitAllDrives)
1717
, UseActorSystemTimeInBSQueue(params.UseActorSystemTimeInBSQueue)
18-
, EnablePutBatching(params.EnablePutBatching)
19-
, EnableVPatch(params.EnableVPatch)
20-
, SlowDiskThreshold(params.SlowDiskThreshold)
21-
, PredictedDelayMultiplier(params.PredictedDelayMultiplier)
22-
, MaxNumOfSlowDisks(params.MaxNumOfSlowDisks)
23-
, LongRequestThresholdMs(params.LongRequestThresholdMs)
18+
, Controls(std::move(params.Controls))
2419
{}
2520

2621
TBlobStorageGroupProxy::TBlobStorageGroupProxy(ui32 groupId, bool isEjected,TIntrusivePtr<TDsProxyNodeMon> &nodeMon,
@@ -30,21 +25,16 @@ namespace NKikimr {
3025
, IsEjected(isEjected)
3126
, ForceWaitAllDrives(false)
3227
, UseActorSystemTimeInBSQueue(params.UseActorSystemTimeInBSQueue)
33-
, EnablePutBatching(params.EnablePutBatching)
34-
, EnableVPatch(params.EnableVPatch)
35-
, SlowDiskThreshold(params.SlowDiskThreshold)
36-
, PredictedDelayMultiplier(params.PredictedDelayMultiplier)
37-
, MaxNumOfSlowDisks(params.MaxNumOfSlowDisks)
38-
, LongRequestThresholdMs(params.LongRequestThresholdMs)
28+
, Controls(std::move(params.Controls))
3929
{}
4030

4131
IActor* CreateBlobStorageGroupEjectedProxy(ui32 groupId, TIntrusivePtr<TDsProxyNodeMon> &nodeMon) {
4232
return new TBlobStorageGroupProxy(groupId, true, nodeMon,
4333
TBlobStorageProxyParameters{
44-
.EnablePutBatching = TControlWrapper(false, false, true),
45-
.EnableVPatch = TControlWrapper(false, false, true),
46-
.SlowDiskThreshold = TControlWrapper(2000, 1, 1000000),
47-
.PredictedDelayMultiplier = TControlWrapper(1000, 1, 1000000),
34+
.Controls = TBlobStorageProxyControlWrappers{
35+
.EnablePutBatching = TControlWrapper(false, false, true),
36+
.EnableVPatch = TControlWrapper(false, false, true),
37+
}
4838
}
4939
);
5040
}

‎ydb/core/blobstorage/dsproxy/dsproxy_impl.h‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,6 @@ class TBlobStorageGroupProxy : public TActorBootstrapped<TBlobStorageGroupProxy>
108108
TBatchedQueue<TEvBlobStorage::TEvGet::TPtr> BatchedGets[GetHandleClassCount];
109109
TStackVec<NKikimrBlobStorage::EGetHandleClass, GetHandleClassCount> GetBatchedBucketQueue;
110110

111-
TMemorizableControlWrapper EnablePutBatching;
112-
TMemorizableControlWrapper EnableVPatch;
113-
114111
TInstant EstablishingSessionStartTime;
115112

116113
const TDuration MuteDuration = TDuration::Seconds(5);
@@ -121,12 +118,7 @@ class TBlobStorageGroupProxy : public TActorBootstrapped<TBlobStorageGroupProxy>
121118
bool HasInvalidGroupId() const { return GroupId.GetRawId() == Max<ui32>(); }
122119
void ProcessInitQueue();
123120

124-
// Acceleration parameters
125-
TMemorizableControlWrapper SlowDiskThreshold;
126-
TMemorizableControlWrapper PredictedDelayMultiplier;
127-
TMemorizableControlWrapper MaxNumOfSlowDisks;
128-
129-
TMemorizableControlWrapper LongRequestThresholdMs;
121+
TBlobStorageProxyControlWrappers Controls;
130122

131123
TAccelerationParams GetAccelerationParams();
132124

‎ydb/core/blobstorage/dsproxy/dsproxy_request.cpp‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ namespace NKikimr {
112112
},
113113
.NodeLayout = TNodeLayoutInfoPtr(NodeLayoutInfo),
114114
.AccelerationParams = GetAccelerationParams(),
115-
.LongRequestThreshold = TDuration::MilliSeconds(LongRequestThresholdMs.Update(TActivationContext::Now())),
115+
.LongRequestThreshold = TDuration::MilliSeconds(Controls.LongRequestThresholdMs.Update(TActivationContext::Now())),
116116
}),
117117
ev->Get()->Deadline
118118
);
@@ -181,7 +181,10 @@ namespace NKikimr {
181181
Y_DEBUG_ABORT_UNLESS(MinREALHugeBlobInBytes);
182182
const ui32 partSize = Info->Type.PartSize(ev->Get()->Id);
183183

184-
if (EnablePutBatching && partSize < MinREALHugeBlobInBytes && partSize <= MaxBatchedPutSize) {
184+
TInstant now = TActivationContext::Now();
185+
186+
if (Controls.EnablePutBatching.Update(now) && partSize < MinREALHugeBlobInBytes &&
187+
partSize <= MaxBatchedPutSize) {
185188
NKikimrBlobStorage::EPutHandleClass handleClass = ev->Get()->HandleClass;
186189
TEvBlobStorage::TEvPut::ETactic tactic = ev->Get()->Tactic;
187190
Y_ABORT_UNLESS((ui64)handleClass <= PutHandleClassCount);
@@ -226,7 +229,7 @@ namespace NKikimr {
226229
.Stats = PerDiskStats,
227230
.EnableRequestMod3x3ForMinLatency = enableRequestMod3x3ForMinLatency,
228231
.AccelerationParams = GetAccelerationParams(),
229-
.LongRequestThreshold = TDuration::MilliSeconds(LongRequestThresholdMs.Update(TActivationContext::Now())),
232+
.LongRequestThreshold = TDuration::MilliSeconds(Controls.LongRequestThresholdMs.Update(now)),
230233
}),
231234
ev->Get()->Deadline
232235
);
@@ -279,7 +282,7 @@ namespace NKikimr {
279282
.Event = ev->Get(),
280283
.ExecutionRelay = ev->Get()->ExecutionRelay
281284
},
282-
.UseVPatch = static_cast<bool>(EnableVPatch.Update(TActivationContext::Now()))
285+
.UseVPatch = static_cast<bool>(Controls.EnableVPatch.Update(TActivationContext::Now()))
283286
}),
284287
ev->Get()->Deadline
285288
);
@@ -500,7 +503,7 @@ namespace NKikimr {
500503
.Stats = PerDiskStats,
501504
.EnableRequestMod3x3ForMinLatency = enableRequestMod3x3ForMinLatency,
502505
.AccelerationParams = GetAccelerationParams(),
503-
.LongRequestThreshold = TDuration::MilliSeconds(LongRequestThresholdMs.Update(TActivationContext::Now())),
506+
.LongRequestThreshold = TDuration::MilliSeconds(Controls.LongRequestThresholdMs.Update(TActivationContext::Now())),
504507
}),
505508
ev->Get()->Deadline
506509
);
@@ -523,7 +526,7 @@ namespace NKikimr {
523526
.Tactic = tactic,
524527
.EnableRequestMod3x3ForMinLatency = enableRequestMod3x3ForMinLatency,
525528
.AccelerationParams = GetAccelerationParams(),
526-
.LongRequestThreshold = TDuration::MilliSeconds(LongRequestThresholdMs.Update(TActivationContext::Now())),
529+
.LongRequestThreshold = TDuration::MilliSeconds(Controls.LongRequestThresholdMs.Update(TActivationContext::Now())),
527530
}),
528531
TInstant::Max()
529532
);
@@ -557,7 +560,7 @@ namespace NKikimr {
557560
++*Mon->EventStopPutBatching;
558561
LWPROBE(DSProxyBatchedPutRequest, BatchedPutRequestCount, GroupId.GetRawId());
559562
BatchedPutRequestCount = 0;
560-
EnablePutBatching.Update(TActivationContext::Now());
563+
Controls.EnablePutBatching.Update(TActivationContext::Now());
561564
}
562565

563566
void TBlobStorageGroupProxy::Handle(TEvStopBatchingGetRequests::TPtr& ev) {

‎ydb/core/blobstorage/dsproxy/dsproxy_state.cpp‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -323,12 +323,28 @@ namespace NKikimr {
323323
Send(ev->Sender, new TEvProxySessionsState(Sessions ? Sessions->GroupQueues : nullptr));
324324
}
325325

326+
#define SELECT_CONTROL_BY_DEVICE_TYPE(prefix, info) \
327+
([&](NPDisk::EDeviceType deviceType) -> i64 { \
328+
TInstant now = TActivationContext::Now(); \
329+
switch (deviceType) { \
330+
case NPDisk::DEVICE_TYPE_ROT: \
331+
return Controls.prefix##HDD.Update(now); \
332+
case NPDisk::DEVICE_TYPE_SSD: \
333+
case NPDisk::DEVICE_TYPE_NVME: \
334+
return Controls.prefix##SSD.Update(now); \
335+
default: \
336+
return Controls.prefix.Update(now); \
337+
} \
338+
})(info ? info->GetDeviceType() : NPDisk::DEVICE_TYPE_UNKNOWN)
339+
326340
TAccelerationParams TBlobStorageGroupProxy::GetAccelerationParams() {
327341
return TAccelerationParams{
328-
.SlowDiskThreshold = .001f * SlowDiskThreshold.Update(TActivationContext::Now()),
329-
.PredictedDelayMultiplier = .001f * PredictedDelayMultiplier.Update(TActivationContext::Now()),
330-
.MaxNumOfSlowDisks = (ui32)MaxNumOfSlowDisks.Update(TActivationContext::Now()),
342+
.SlowDiskThreshold = .001f * SELECT_CONTROL_BY_DEVICE_TYPE(SlowDiskThreshold, Info),
343+
.PredictedDelayMultiplier = .001f * SELECT_CONTROL_BY_DEVICE_TYPE(PredictedDelayMultiplier, Info),
344+
.MaxNumOfSlowDisks = static_cast<ui32>(SELECT_CONTROL_BY_DEVICE_TYPE(MaxNumOfSlowDisks, Info)),
331345
};
332346
}
333347

348+
#undef SELECT_CONTROL_BY_DEVICE_TYPE
349+
334350
} // NKikimr

‎ydb/core/blobstorage/dsproxy/ut/dsproxy_env_mock_ut.h‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,12 @@ struct TDSProxyEnv {
8181
TIntrusivePtr<TStoragePoolCounters> storagePoolCounters = perPoolCounters.GetPoolCounters("pool_name");
8282
TControlWrapper enablePutBatching(DefaultEnablePutBatching, false, true);
8383
TControlWrapper enableVPatch(DefaultEnableVPatch, false, true);
84-
TControlWrapper slowDiskThreshold(DefaultSlowDiskThreshold * 1000, 1, 1000000);
85-
TControlWrapper predictedDelayMultiplier(DefaultPredictedDelayMultiplier * 1000, 1, 1000000);
8684
IActor *dsproxy = CreateBlobStorageGroupProxyConfigured(TIntrusivePtr(Info), true, nodeMon,
8785
std::move(storagePoolCounters), TBlobStorageProxyParameters{
88-
.EnablePutBatching = enablePutBatching,
89-
.EnableVPatch = enableVPatch,
90-
.SlowDiskThreshold = slowDiskThreshold,
91-
.PredictedDelayMultiplier = predictedDelayMultiplier,
86+
.Controls = TBlobStorageProxyControlWrappers{
87+
.EnablePutBatching = enablePutBatching,
88+
.EnableVPatch = enableVPatch,
89+
}
9290
}
9391
);
9492
TActorId actorId = runtime.Register(dsproxy, nodeIndex);

0 commit comments

Comments
 (0)