Skip to content

Report inferred PDisk SlotCount in metrics #21115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions ydb/apps/dstool/lib/dstool_cmd_pdisk_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,15 @@ def do(args):
row['Kind'] = pdisk.Kind
row['Guid'] = pdisk.Guid
row['NumStaticSlots'] = pdisk.NumStaticSlots
row['ExpectedSlotCount'] = pdisk.ExpectedSlotCount
row['SlotSizeInUnits'] = pdisk.PDiskConfig.SlotSizeInUnits
if (pdisk.PDiskMetrics.HasField('SlotCount')):
row['ExpectedSlotCount'] = pdisk.PDiskMetrics.SlotCount
row['SlotSizeInUnits'] = pdisk.PDiskMetrics.SlotSizeInUnits
elif (pdisk.InferPDiskSlotCountFromUnitSize != 0):
row['ExpectedSlotCount'] = 0
row['SlotSizeInUnits'] = 0
else:
row['ExpectedSlotCount'] = pdisk.ExpectedSlotCount
row['SlotSizeInUnits'] = pdisk.PDiskConfig.SlotSizeInUnits
row['PDiskConfig'] = text_format.MessageToString(pdisk.PDiskConfig, as_one_line=True)
row['AvailableSize'] = pdisk.PDiskMetrics.AvailableSize
row['TotalSize'] = pdisk.PDiskMetrics.TotalSize
Expand Down
38 changes: 32 additions & 6 deletions ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) {
}

ui32 CreatePDisk(TTestActorRuntime &runtime, ui32 nodeIdx, TString path, ui64 guid, ui32 pdiskId, ui64 pDiskCategory,
ui64 inferPDiskSlotCountFromUnitSize = 0) {
const NKikimrBlobStorage::TPDiskConfig* pdiskConfig = nullptr, ui64 inferPDiskSlotCountFromUnitSize = 0) {
VERBOSE_COUT(" Creating pdisk");

ui32 nodeId = runtime.GetNodeId(nodeIdx);
Expand All @@ -278,7 +278,10 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) {
pdisk->SetPDiskGuid(guid);
pdisk->SetPDiskCategory(pDiskCategory);
pdisk->SetEntityStatus(NKikimrBlobStorage::CREATE);
if (inferPDiskSlotCountFromUnitSize) {
if (pdiskConfig) {
pdisk->MutablePDiskConfig()->CopyFrom(*pdiskConfig);
}
if (inferPDiskSlotCountFromUnitSize != 0) {
pdisk->SetInferPDiskSlotCountFromUnitSize(inferPDiskSlotCountFromUnitSize);
}

Expand Down Expand Up @@ -1005,19 +1008,42 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) {
TTestBasicRuntime runtime(1, false);
auto nodeId = runtime.GetNodeId(0);
TString pdiskPath = "SectorMap:TestInferPDiskSlotCountComplexSetup";
TIntrusivePtr<NPDisk::TSectorMap> sectorMap(new NPDisk::TSectorMap(32_GB));
TIntrusivePtr<NPDisk::TSectorMap> sectorMap(new NPDisk::TSectorMap(30_GB));
Setup(runtime, pdiskPath, sectorMap);

TActorId edge = runtime.AllocateEdgeActor();
runtime.SetDispatchTimeout(TDuration::Seconds(10));
runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), edge);

NKikimrBlobStorage::TPDiskConfig pdiskConfig;
ui32 pdiskId = CreatePDisk(runtime, 0, pdiskPath, 0, 1001, 0, 1_GB);
ui32 pdiskId = CreatePDisk(runtime, 0, pdiskPath, 0, 1001, 0, nullptr, 1_GB);
std::optional<NKikimrWhiteboard::TPDiskStateInfo> pdiskInfo = GrabEvPDiskStateUpdate(runtime, edge, pdiskId);
UNIT_ASSERT_C(pdiskInfo, "No appropriate TEvPDiskStateUpdate received");
UNIT_ASSERT_VALUES_EQUAL(pdiskInfo->GetExpectedSlotCount(), 16);
UNIT_ASSERT_VALUES_EQUAL(pdiskInfo->GetExpectedSlotCount(), 15);
UNIT_ASSERT_VALUES_EQUAL(pdiskInfo->GetSlotSizeInUnits(), 2u);

}

CUSTOM_UNIT_TEST(TestInferPDiskSlotCountExplicitConfig) {
TTempDir tempDir;

TTestBasicRuntime runtime(1, false);
auto nodeId = runtime.GetNodeId(0);
TString pdiskPath = "SectorMap:TestInferPDiskSlotCountExplicitConfig";
TIntrusivePtr<NPDisk::TSectorMap> sectorMap(new NPDisk::TSectorMap(30_GB));
Setup(runtime, pdiskPath, sectorMap);

NKikimrBlobStorage::TPDiskConfig pdiskConfig;
pdiskConfig.SetExpectedSlotCount(13);
ui32 pdiskId = CreatePDisk(runtime, 0, pdiskPath, 0, 1001, 0, &pdiskConfig, 512_MB);

TActorId edge = runtime.AllocateEdgeActor();
runtime.SetDispatchTimeout(TDuration::Seconds(10));
runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), edge);

std::optional<NKikimrWhiteboard::TPDiskStateInfo> pdiskInfo = GrabEvPDiskStateUpdate(runtime, edge, pdiskId);
UNIT_ASSERT_C(pdiskInfo, "No appropriate TEvPDiskStateUpdate received");
UNIT_ASSERT_VALUES_EQUAL(pdiskInfo->GetExpectedSlotCount(), 13);
UNIT_ASSERT_VALUES_EQUAL(pdiskInfo->GetSlotSizeInUnits(), 0u);
}
}

Expand Down
4 changes: 3 additions & 1 deletion ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,9 @@ namespace NKikimr::NStorage {
pdiskConfig->EnableSectorEncryption = !pdiskConfig->SectorMap;
}

if (ui64 unitSizeInBytes = pdisk.GetInferPDiskSlotCountFromUnitSize()) {
if (pdisk.GetPDiskConfig().GetExpectedSlotCount() != 0) {
// Skip inferring PDisk SlotCount
} else if (ui64 unitSizeInBytes = pdisk.GetInferPDiskSlotCountFromUnitSize()) {
ui64 driveSize = 0;
TStringStream outDetails;
if (pdiskConfig->SectorMap) {
Expand Down
14 changes: 9 additions & 5 deletions ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1630,6 +1630,10 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) {
pdiskState.SetEnforcedDynamicSlotSize(minSlotSize);
}
pDiskMetrics.SetState(state);
pDiskMetrics.SetSlotSizeInUnits(Cfg->SlotSizeInUnits);
if (ExpectedSlotCount) {
pDiskMetrics.SetSlotCount(ExpectedSlotCount);
}
}

PCtx->ActorSystem->Send(whiteboardReport.Sender, reportResult);
Expand Down Expand Up @@ -2330,7 +2334,7 @@ void TPDisk::Slay(TSlay &evSlay) {
TVDiskID vDiskId = evSlay.VDiskId;
vDiskId.GroupGeneration = -1;
auto it = VDiskOwners.find(vDiskId);

for (auto& pendingInit : PendingYardInits) {
if (vDiskId == pendingInit->VDiskIdWOGeneration()) {
TStringStream str;
Expand Down Expand Up @@ -2999,7 +3003,7 @@ NKikimrProto::EReplyStatus TPDisk::CheckOwnerAndRound(TRequestBase* req, TString

if (!IsOwnerUser(req->Owner)) {
if (req->Owner == OwnerUnallocated && req->OwnerRound == 0) {
return NKikimrProto::OK;
return NKikimrProto::OK;
}
err << " ownerId# " << req->Owner << " < Begin# " << (ui32)OwnerBeginUser
<< " or >= End# " << (ui32)OwnerEndUser << " Marker# BPD72";
Expand Down Expand Up @@ -4343,7 +4347,7 @@ void TPDisk::ProgressShredState() {
<< " ShredGeneration# " << ShredGeneration
<< " ShredState# " << (ui32)ShredState);
// Send/schedule a request to retry
THolder<TCompletionEventSender> completion(new TCompletionEventSender(this, PCtx->PDiskActor, new NPDisk::TEvContinueShred()));
THolder<TCompletionEventSender> completion(new TCompletionEventSender(this, PCtx->PDiskActor, new NPDisk::TEvContinueShred()));
if (ReleaseUnusedLogChunks(completion.Get())) {
ContinueShredsInFlight++;
WriteSysLogRestorePoint(completion.Release(), TReqId(TReqId::ShredPDisk, 0), {});
Expand All @@ -4367,7 +4371,7 @@ void TPDisk::ProgressShredState() {
}
}
// Looks good, but there still can be chunks that need to be shredded still int transition between states.
// For example, log chunks are removed from the log chunk list on log cut but added to free chunk list on log cut
// For example, log chunks are removed from the log chunk list on log cut but added to free chunk list on log cut
// write operation completion. So, walk through the whole chunk list and check.
for (ui32 chunkIdx = Format.SystemChunkCount; chunkIdx < ChunkState.size(); ++chunkIdx) {
TChunkState &state = ChunkState[chunkIdx];
Expand All @@ -4382,7 +4386,7 @@ void TPDisk::ProgressShredState() {
<< ", there are already ContinueShredsInFlight# " << ContinueShredsInFlight.load()
<< " so just wait for it to arrive. "
<< " ShredGeneration# " << ShredGeneration);
return;
return;
} else {
LOG_DEBUG_S(*PCtx->ActorSystem, NKikimrServices::BS_PDISK_SHRED,
"PDisk# " << PCtx->PDiskId
Expand Down
18 changes: 16 additions & 2 deletions ydb/core/mind/bscontroller/config_fit_groups.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -678,14 +678,28 @@ namespace NKikimr {
}
}

ui32 maxSlots = 0;
ui32 slotSizeInUnits = 0;
if (info.Metrics.HasSlotCount()) {
maxSlots = info.Metrics.GetSlotCount();
slotSizeInUnits = info.Metrics.GetSlotSizeInUnits();
} else if (info.InferPDiskSlotCountFromUnitSize != 0) {
// inferred values are unknown yet
maxSlots = 0;
slotSizeInUnits = 0;
} else {
maxSlots = info.ExpectedSlotCount;
slotSizeInUnits = info.SlotSizeInUnits;
}

// register PDisk in the mapper
return Mapper->RegisterPDisk({
.PDiskId = id,
.Location = State.HostRecords->GetLocation(id.NodeId),
.Usable = usable,
.NumSlots = numSlots,
.MaxSlots = info.ExpectedSlotCount,
.SlotSizeInUnits = info.SlotSizeInUnits,
.MaxSlots = maxSlots,
.SlotSizeInUnits = slotSizeInUnits,
.Groups = std::move(groups),
.SpaceAvailable = availableSpace,
.Operational = info.Operational,
Expand Down
6 changes: 5 additions & 1 deletion ydb/core/mind/bscontroller/group_mapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,11 @@ namespace NKikimr::NBsController {
// single-unit vdisk occupies double-unit pdisk slot (storage waste)
penalty += 20;
}
return double(NumSlots) / MaxSlots + penalty;
if (!MaxSlots) {
return NumSlots + penalty;
} else {
return double(NumSlots) / MaxSlots + penalty;
}
}
};

Expand Down
4 changes: 3 additions & 1 deletion ydb/core/protos/blobstorage_disk.proto
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,14 @@ message TPDiskMetrics {
optional uint64 NonRealTimeMs = 6; // ms per second of non-realtime, [0..1000]
optional uint64 SlowDeviceMs = 7; // ms per second of slow device, [0..1000]
optional uint32 MaxIOPS = 8; // number of IOPS this device can carry out

// maximum expected slot size; if set, then it is guaranteed that EnforcedDynamicSlotSize bytes of space are
// available for every slot over this PDisk; if not set, then no space enforcement assumed
optional uint64 EnforcedDynamicSlotSize = 9;

optional TPDiskState.E State = 10;

optional uint64 UpdateTimestamp = 11; // TInstant::GetValue()
optional uint64 SlotCount = 12;
optional uint32 SlotSizeInUnits = 13;
}
Loading