Skip to content

Commit 21448f2

Browse files
serbel324serbel324
authored
Merge 22e9603 into ec1169f 
2 parents ec1169f + 22e9603 commit 21448f2

40 files changed

+901
-377
lines changed

‎library/cpp/bucket_quoter/bucket_quoter.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,8 @@ class TBucketQuoter {
172172

173173
i64 UseAndFill(ui64 tokens) {
174174
TGuard<Lock> g(BucketMutex);
175-
UseNoLock(tokens);
176175
FillBucket();
176+
UseNoLock(tokens);
177177
return Bucket;
178178
}
179179

‎ydb/core/base/blobstorage_grouptype.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,11 @@ ui64 TBlobStorageGroupType::PartSize(const TLogoBlobID &id) const {
196196

197197
ui64 TBlobStorageGroupType::MaxPartSize(const TLogoBlobID &id) const {
198198
Y_ABORT_UNLESS(!id.PartId());
199-
return TErasureType::PartSize((TErasureType::ECrcMode)id.CrcMode(), id.BlobSize());
199+
return MaxPartSize((TErasureType::ECrcMode)id.CrcMode(), id.BlobSize());
200+
}
201+
202+
ui64 TBlobStorageGroupType::MaxPartSize(TErasureType::ECrcMode crcMode, ui32 blobSize) const {
203+
return TErasureType::PartSize(crcMode, blobSize);
200204
}
201205

202206
bool TBlobStorageGroupType::PartFits(ui32 partId, ui32 idxInSubgroup) const {

‎ydb/core/base/blobstorage_grouptype.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ struct TBlobStorageGroupType : public TErasureType {
144144

145145
ui64 PartSize(const TLogoBlobID &id) const;
146146
ui64 MaxPartSize(const TLogoBlobID &id) const;
147+
ui64 MaxPartSize(TErasureType::ECrcMode crcMode, ui32 blobSize) const;
147148

148149
bool PartFits(ui32 partId, ui32 idxInSubgroup) const;
149150
};

‎ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4252,6 +4252,7 @@ class TBlobStorageProxyTest: public TTestBase {
42524252
vDiskConfig->GCOnlySynced = false;
42534253
vDiskConfig->HullCompLevelRateThreshold = 0.1;
42544254
vDiskConfig->SkeletonFrontQueueBackpressureCheckMsgId = false;
4255+
vDiskConfig->UseCostTracker = false;
42554256

42564257
IActor* vDisk = CreateVDisk(vDiskConfig, bsInfo, counters);
42574258
TActorSetupCmd vDiskSetup(vDisk, TMailboxType::Revolving, 0);

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,28 @@ void TNodeWarden::Bootstrap() {
171171
if (actorSystem && actorSystem->AppData<TAppData>() && actorSystem->AppData<TAppData>()->Icb) {
172172
actorSystem->AppData<TAppData>()->Icb->RegisterLocalControl(EnablePutBatching, "BlobStorage_EnablePutBatching");
173173
actorSystem->AppData<TAppData>()->Icb->RegisterLocalControl(EnableVPatch, "BlobStorage_EnableVPatch");
174+
const TIntrusivePtr<NKikimr::TControlBoard>& icb = actorSystem->AppData<TAppData>()->Icb;
175+
176+
icb->RegisterLocalControl(EnablePutBatching, "BlobStorage_EnablePutBatching");
177+
icb->RegisterLocalControl(EnableVPatch, "BlobStorage_EnableVPatch");
178+
icb->RegisterSharedControl(EnableLocalSyncLogDataCutting, "VDiskControls.EnableLocalSyncLogDataCutting");
179+
icb->RegisterSharedControl(EnableSyncLogChunkCompressionHDD, "VDiskControls.EnableSyncLogChunkCompressionHDD");
180+
icb->RegisterSharedControl(EnableSyncLogChunkCompressionSSD, "VDiskControls.EnableSyncLogChunkCompressionSSD");
181+
icb->RegisterSharedControl(MaxSyncLogChunksInFlightHDD, "VDiskControls.MaxSyncLogChunksInFlightHDD");
182+
icb->RegisterSharedControl(MaxSyncLogChunksInFlightSSD, "VDiskControls.MaxSyncLogChunksInFlightSSD");
183+
184+
icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_ROT].BurstThresholdNs,
185+
"VDiskControls.BurstThresholdNsHDD");
186+
icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_SSD].BurstThresholdNs,
187+
"VDiskControls.BurstThresholdNsSSD");
188+
icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_NVME].BurstThresholdNs,
189+
"VDiskControls.BurstThresholdNsNVME");
190+
icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_ROT].DiskTimeAvailableScale,
191+
"VDiskControls.DiskTimeAvailableScaleHDD");
192+
icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_SSD].DiskTimeAvailableScale,
193+
"VDiskControls.DiskTimeAvailableScaleSSD");
194+
icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_NVME].DiskTimeAvailableScale,
195+
"VDiskControls.DiskTimeAvailableScaleNVME");
174196
}
175197

176198
// start replication broker

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ namespace NKikimr::NStorage {
125125
TReplQuoter::TPtr ReplNodeRequestQuoter;
126126
TReplQuoter::TPtr ReplNodeResponseQuoter;
127127

128+
TCostMetricsParametersByMedia CostMetricsParametersByMedia;
129+
128130
public:
129131
struct TGroupRecord;
130132

@@ -137,6 +139,16 @@ namespace NKikimr::NStorage {
137139
: Cfg(cfg)
138140
, EnablePutBatching(Cfg->FeatureFlags.GetEnablePutBatchingForBlobStorage(), false, true)
139141
, EnableVPatch(Cfg->FeatureFlags.GetEnableVPatch(), false, true)
142+
, EnableLocalSyncLogDataCutting(0, 0, 1)
143+
, EnableSyncLogChunkCompressionHDD(1, 0, 1)
144+
, EnableSyncLogChunkCompressionSSD(0, 0, 1)
145+
, MaxSyncLogChunksInFlightHDD(10, 1, 1024)
146+
, MaxSyncLogChunksInFlightSSD(10, 1, 1024)
147+
, CostMetricsParametersByMedia({
148+
TCostMetricsParameters{200},
149+
TCostMetricsParameters{50},
150+
TCostMetricsParameters{32},
151+
})
140152
{
141153
Y_ABORT_UNLESS(Cfg->BlobStorageConfig.GetServiceSet().AvailabilityDomainsSize() <= 1);
142154
AvailDomainId = 1;

‎ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,18 @@ namespace NKikimr::NStorage {
174174
vdiskConfig->EnableVDiskCooldownTimeout = Cfg->EnableVDiskCooldownTimeout;
175175
vdiskConfig->ReplPausedAtStart = Cfg->VDiskReplPausedAtStart;
176176
vdiskConfig->EnableVPatch = EnableVPatch;
177+
178+
vdiskConfig->EnableLocalSyncLogDataCutting = EnableLocalSyncLogDataCutting;
179+
if (deviceType == NPDisk::EDeviceType::DEVICE_TYPE_ROT) {
180+
vdiskConfig->EnableSyncLogChunkCompression = EnableSyncLogChunkCompressionHDD;
181+
vdiskConfig->MaxSyncLogChunksInFlight = MaxSyncLogChunksInFlightHDD;
182+
} else {
183+
vdiskConfig->EnableSyncLogChunkCompression = EnableSyncLogChunkCompressionSSD;
184+
vdiskConfig->MaxSyncLogChunksInFlight = MaxSyncLogChunksInFlightSSD;
185+
}
186+
187+
vdiskConfig->CostMetricsParametersByMedia = CostMetricsParametersByMedia;
188+
177189
vdiskConfig->FeatureFlags = Cfg->FeatureFlags;
178190

179191
// issue initial report to whiteboard before creating actor to avoid races

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ struct TEvYardInitResult : public TEventLocal<TEvYardInitResult, TEvBlobStorage:
173173
TEvYardInitResult(const NKikimrProto::EReplyStatus status, const TString &errorReason)
174174
: Status(status)
175175
, StatusFlags(0)
176-
, PDiskParams(new TPDiskParams(0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
176+
, PDiskParams(new TPDiskParams(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, DEVICE_TYPE_ROT))
177177
, ErrorReason(errorReason)
178178
{
179179
Y_ABORT_UNLESS(status != NKikimrProto::OK, "Single-parameter constructor is for error responses only");
@@ -183,7 +183,7 @@ struct TEvYardInitResult : public TEventLocal<TEvYardInitResult, TEvBlobStorage:
183183
ui64 writeSpeedBps, ui64 readBlockSize, ui64 writeBlockSize,
184184
ui64 bulkWriteBlockSize, ui32 chunkSize, ui32 appendBlockSize,
185185
TOwner owner, TOwnerRound ownerRound, TStatusFlags statusFlags, TVector<TChunkIdx> ownedChunks,
186-
const TString &errorReason)
186+
EDeviceType trueMediaType, const TString &errorReason)
187187
: Status(status)
188188
, StatusFlags(statusFlags)
189189
, PDiskParams(new TPDiskParams(
@@ -196,8 +196,8 @@ struct TEvYardInitResult : public TEventLocal<TEvYardInitResult, TEvBlobStorage:
196196
writeSpeedBps,
197197
readBlockSize,
198198
writeBlockSize,
199-
bulkWriteBlockSize
200-
))
199+
bulkWriteBlockSize,
200+
trueMediaType))
201201
, OwnedChunks(std::move(ownedChunks))
202202
, ErrorReason(errorReason)
203203
{}

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1769,7 +1769,8 @@ void TPDisk::ReplyErrorYardInitResult(TYardInit &evYardInit, const TString &str)
17691769
DriveModel.Speed(TDriveModel::OP_TYPE_WRITE), readBlockSize, writeBlockSize,
17701770
DriveModel.BulkWriteBlockSize(),
17711771
GetUserAccessibleChunkSize(), GetChunkAppendBlockSize(), OwnerSystem, 0,
1772-
GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), TVector<TChunkIdx>(), error.Str()));
1772+
GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), TVector<TChunkIdx>(),
1773+
Cfg->RetrieveDeviceType(), error.Str()));
17731774
Mon.YardInit.CountResponse();
17741775
}
17751776

@@ -1816,7 +1817,8 @@ bool TPDisk::YardInitForKnownVDisk(TYardInit &evYardInit, TOwner owner) {
18161817
DriveModel.SeekTimeNs() / 1000ull, DriveModel.Speed(TDriveModel::OP_TYPE_READ),
18171818
DriveModel.Speed(TDriveModel::OP_TYPE_WRITE), readBlockSize, writeBlockSize,
18181819
DriveModel.BulkWriteBlockSize(), GetUserAccessibleChunkSize(), GetChunkAppendBlockSize(), owner,
1819-
ownerRound, GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), ownedChunks, nullptr));
1820+
ownerRound, GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), ownedChunks,
1821+
Cfg->RetrieveDeviceType(), nullptr));
18201822
GetStartingPoints(owner, result->StartingPoints);
18211823
ownerData.VDiskId = vDiskId;
18221824
ownerData.CutLogId = evYardInit.CutLogId;
@@ -1967,7 +1969,7 @@ void TPDisk::YardInitFinish(TYardInit &evYardInit) {
19671969
DriveModel.Speed(TDriveModel::OP_TYPE_WRITE), readBlockSize, writeBlockSize,
19681970
DriveModel.BulkWriteBlockSize(), GetUserAccessibleChunkSize(), GetChunkAppendBlockSize(), owner, ownerRound,
19691971
GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType) | ui32(NKikimrBlobStorage::StatusNewOwner), TVector<TChunkIdx>(),
1970-
nullptr));
1972+
Cfg->RetrieveDeviceType(), nullptr));
19711973
GetStartingPoints(result->PDiskParams->Owner, result->StartingPoints);
19721974
WriteSysLogRestorePoint(new TCompletionEventSender(
19731975
this, evYardInit.Sender, result.Release(), Mon.YardInit.Results), evYardInit.ReqId, {});

0 commit comments

Comments
 (0)