Skip to content

Commit 02662eb

Browse files
SammyVimesSammyVimes
authored
Add node interlace in TScore for group mapper (#13867)
1 parent dead33f commit 02662eb

File tree

3 files changed

+74
-14
lines changed

3 files changed

+74
-14
lines changed

‎ydb/core/mind/bscontroller/group_layout_checker.h

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -72,21 +72,25 @@ namespace NKikimr::NBsController {
7272
TEntityId RealmGroup;
7373
TEntityId Realm;
7474
TEntityId Domain;
75+
TEntityId Device;
7576

7677
TPDiskLayoutPosition() = default;
7778

78-
TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain)
79+
TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain, TEntityId device)
7980
: RealmGroup(realmGroup)
8081
, Realm(realm)
8182
, Domain(domain)
83+
, Device(device)
8284
{}
8385

8486
TPDiskLayoutPosition(TDomainMapper& mapper, const TNodeLocation& location, TPDiskId pdiskId, const TGroupGeometryInfo& geom) {
85-
TStringStream realmGroup, realm, domain;
87+
TStringStream realmGroup, realm, domain, device;
88+
ui32 deviceLevelEnd = TNodeLocation::TKeys::E::Unit + 1;
8689
const std::pair<int, TStringStream*> levels[] = {
8790
{geom.GetRealmLevelBegin(), &realmGroup},
8891
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelBegin()), &realm},
89-
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelEnd()), &domain}
92+
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelEnd()), &domain},
93+
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelEnd(), deviceLevelEnd), &device}
9094
};
9195
auto addLevel = [&](int key, const TString& value) {
9296
for (const auto& [reference, stream] : levels) {
@@ -102,14 +106,15 @@ namespace NKikimr::NBsController {
102106
RealmGroup = mapper(realmGroup.Str());
103107
Realm = mapper(realm.Str());
104108
Domain = mapper(domain.Str());
109+
Device = mapper(device.Str());
105110
}
106111

107112
TString ToString() const {
108-
return TStringBuilder() << "{" << RealmGroup << "." << Realm << "." << Domain << "}";
113+
return TStringBuilder() << "{" << RealmGroup << "." << Realm << "." << Domain << "." << Device << "}";
109114
}
110115

111116
auto AsTuple() const {
112-
return std::tie(RealmGroup, Realm, Domain);
117+
return std::tie(RealmGroup, Realm, Domain, Device);
113118
}
114119

115120
friend bool operator ==(const TPDiskLayoutPosition& x, const TPDiskLayoutPosition& y) {
@@ -124,12 +129,13 @@ namespace NKikimr::NBsController {
124129
struct TScore {
125130
ui32 RealmInterlace = 0;
126131
ui32 DomainInterlace = 0;
132+
ui32 DeviceInterlace = 0;
127133
ui32 RealmGroupScatter = 0;
128134
ui32 RealmScatter = 0;
129135
ui32 DomainScatter = 0;
130136

131137
auto AsTuple() const {
132-
return std::make_tuple(RealmInterlace, DomainInterlace, RealmGroupScatter, RealmScatter, DomainScatter);
138+
return std::make_tuple(RealmInterlace, DomainInterlace, DeviceInterlace, RealmGroupScatter, RealmScatter, DomainScatter);
133139
}
134140

135141
bool BetterThan(const TScore& other) const {
@@ -141,12 +147,13 @@ namespace NKikimr::NBsController {
141147
}
142148

143149
static TScore Max() {
144-
return {::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>()};
150+
return {::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>()};
145151
}
146152

147153
TString ToString() const {
148154
return TStringBuilder() << "{RealmInterlace# " << RealmInterlace
149155
<< " DomainInterlace# " << DomainInterlace
156+
<< " DeviceInterlace# " << DeviceInterlace
150157
<< " RealmGroupScatter# " << RealmGroupScatter
151158
<< " RealmScatter# " << RealmScatter
152159
<< " DomainScatter# " << DomainScatter
@@ -168,6 +175,8 @@ namespace NKikimr::NBsController {
168175
TStackVec<THashMap<TEntityId, ui32>, 32> NumDisksPerDomain;
169176
THashMap<TEntityId, ui32> NumDisksPerDomainTotal;
170177

178+
THashMap<TEntityId, ui32> NumDisksPerDevice;
179+
171180
TGroupLayout(const TBlobStorageGroupInfo::TTopology& topology)
172181
: Topology(topology)
173182
, NumDisksInRealm(Topology.GetTotalFailRealmsNum())
@@ -187,6 +196,8 @@ namespace NKikimr::NBsController {
187196
NumDisksInDomain[domainIdx] += value;
188197
NumDisksPerDomain[domainIdx][pos.Domain] += value;
189198
NumDisksPerDomainTotal[pos.Domain] += value;
199+
200+
NumDisksPerDevice[pos.Device] += value;
190201
}
191202

192203
void AddDisk(const TPDiskLayoutPosition& pos, ui32 orderNumber) {
@@ -204,9 +215,12 @@ namespace NKikimr::NBsController {
204215
const auto& disksPerRealm = NumDisksPerRealm[vdisk.FailRealm][pos.Realm];
205216
const auto& disksPerDomain = NumDisksPerDomain[domainIdx][pos.Domain];
206217

218+
const ui32 disksOnDevice = NumDisksPerDevice[pos.Device];
219+
207220
return {
208221
.RealmInterlace = NumDisksPerRealmTotal[pos.Realm] - disksPerRealm,
209222
.DomainInterlace = NumDisksPerDomainTotal[pos.Domain] - disksPerDomain,
223+
.DeviceInterlace = disksOnDevice,
210224
.RealmGroupScatter = NumDisks - NumDisksPerRealmGroup[pos.RealmGroup],
211225
.RealmScatter = NumDisksInRealm[vdisk.FailRealm] - disksPerRealm,
212226
.DomainScatter = NumDisksInDomain[domainIdx] - disksPerDomain,

‎ydb/core/mind/bscontroller/group_mapper.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,7 @@ namespace NKikimr::NBsController {
395395

396396
static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
397397
scope = x.Domain;
398-
return {x, x};
398+
return {{x.RealmGroup, x.Realm, x.Domain, TEntityId::Min()}, {x.RealmGroup, x.Realm, x.Domain, TEntityId::Max()}};
399399
}
400400
};
401401

@@ -405,7 +405,7 @@ namespace NKikimr::NBsController {
405405

406406
static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
407407
scope = x.Realm;
408-
return {{x.RealmGroup, x.Realm, TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max()}};
408+
return {{x.RealmGroup, x.Realm, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max(), TEntityId::Max()}};
409409
}
410410
};
411411

@@ -415,7 +415,7 @@ namespace NKikimr::NBsController {
415415

416416
static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
417417
scope = x.RealmGroup;
418-
return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max()}};
418+
return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max(), TEntityId::Max()}};
419419
}
420420
};
421421

‎ydb/core/mind/bscontroller/group_mapper_ut.cpp

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -593,10 +593,6 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
593593
TestBlock42(1);
594594
}
595595

596-
Y_UNIT_TEST(Block42_2disk) {
597-
TestBlock42(2);
598-
}
599-
600596
Y_UNIT_TEST(Mirror3dc) {
601597
TTestContext context(6, 3, 3, 3, 3);
602598
TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureMirror3dc));
@@ -614,6 +610,23 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
614610
context.CheckIfGroupsAreMappedCompact();
615611
}
616612

613+
Y_UNIT_TEST(Mirror3dc3Nodes) {
614+
// Each node has 3 disks.
615+
TTestContext context(
616+
{
617+
{1, 1, 1, 1, 3},
618+
{2, 1, 2, 1, 3},
619+
{3, 1, 3, 1, 3},
620+
}
621+
);
622+
623+
TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureMirror3dc, 3, 3, 1, 10, 20, 10, 256));
624+
context.PopulateGroupMapper(mapper, 9);
625+
626+
TGroupMapper::TGroupDefinition group;
627+
UNIT_ASSERT_UNEQUAL(0, context.AllocateGroup(mapper, group));
628+
}
629+
617630
Y_UNIT_TEST(NonUniformCluster) {
618631
std::vector<std::tuple<ui32, ui32, ui32, ui32, ui32>> disks;
619632
for (ui32 rack = 0, body = 0; rack < 12; ++rack) {
@@ -638,6 +651,39 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
638651
}
639652
}
640653

654+
Y_UNIT_TEST(InterlacedRacksWithoutInterlacedNodes) {
655+
TTestContext context(
656+
{
657+
{1, 1, 1, 1, 1}, // node 1
658+
{1, 1, 2, 2, 1},
659+
{1, 1, 3, 3, 2}, // node 3 has two disks
660+
{1, 1, 4, 4, 1},
661+
{1, 1, 5, 5, 1},
662+
{1, 1, 6, 6, 1},
663+
{1, 1, 2, 7, 1}, // node 7 is in the same rack as node 2
664+
{1, 1, 8, 8, 1},
665+
{1, 1, 3, 9, 1}, // node 9 is in the same rack as node 3
666+
}
667+
);
668+
669+
TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::Erasure4Plus2Block));
670+
context.PopulateGroupMapper(mapper, 8);
671+
672+
TGroupMapper::TGroupDefinition group;
673+
group.emplace_back(TVector<TVector<TPDiskId>>(8));
674+
auto& g = group[0];
675+
676+
for (int i = 0; i < 8; i++) {
677+
g[i].emplace_back(TPDiskId(i + 1, 1));
678+
}
679+
680+
context.SetGroup(1, group);
681+
682+
TGroupMapper::TGroupDefinition newGroup = context.ReallocateGroup(mapper, 1, {TPDiskId(8, 1)});
683+
684+
UNIT_ASSERT_EQUAL_C(TPDiskId(9, 1), newGroup[0][7][0], context.FormatGroup(newGroup));
685+
}
686+
641687
Y_UNIT_TEST(NonUniformClusterDifferentSlotsPerDisk) {
642688
std::vector<std::tuple<ui32, ui32, ui32, ui32, ui32>> disks;
643689
for (ui32 rack = 0; rack < 12; ++rack) {

0 commit comments

Comments
 (0)