Skip to content

Commit 2d8bafb

Browse files
serbel324SammyVimes
serbel324
and
SammyVimes
authored
Add node interlace in TScore for group mapper (#13867) (#15950)
Co-authored-by: Semyon Danilov <senya@ydb.tech>
1 parent da131a0 commit 2d8bafb

File tree

3 files changed

+81
-18
lines changed

3 files changed

+81
-18
lines changed

‎ydb/core/mind/bscontroller/group_layout_checker.h

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -72,21 +72,25 @@ namespace NKikimr::NBsController {
7272
TEntityId RealmGroup;
7373
TEntityId Realm;
7474
TEntityId Domain;
75+
TEntityId Device;
7576

7677
TPDiskLayoutPosition() = default;
7778

78-
TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain)
79+
TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain, TEntityId device)
7980
: RealmGroup(realmGroup)
8081
, Realm(realm)
8182
, Domain(domain)
83+
, Device(device)
8284
{}
8385

8486
TPDiskLayoutPosition(TDomainMapper& mapper, const TNodeLocation& location, TPDiskId pdiskId, const TGroupGeometryInfo& geom) {
85-
TStringStream realmGroup, realm, domain;
87+
TStringStream realmGroup, realm, domain, device;
88+
ui32 deviceLevelEnd = TNodeLocation::TKeys::E::Unit + 1;
8689
const std::pair<int, TStringStream*> levels[] = {
8790
{geom.GetRealmLevelBegin(), &realmGroup},
8891
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelBegin()), &realm},
89-
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelEnd()), &domain}
92+
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelEnd()), &domain},
93+
{Max(geom.GetRealmLevelEnd(), geom.GetDomainLevelEnd(), deviceLevelEnd), &device}
9094
};
9195
auto addLevel = [&](int key, const TString& value) {
9296
for (const auto& [reference, stream] : levels) {
@@ -102,14 +106,15 @@ namespace NKikimr::NBsController {
102106
RealmGroup = mapper(realmGroup.Str());
103107
Realm = mapper(realm.Str());
104108
Domain = mapper(domain.Str());
109+
Device = mapper(device.Str());
105110
}
106111

107112
TString ToString() const {
108-
return TStringBuilder() << "{" << RealmGroup << "." << Realm << "." << Domain << "}";
113+
return TStringBuilder() << "{" << RealmGroup << "." << Realm << "." << Domain << "." << Device << "}";
109114
}
110115

111116
auto AsTuple() const {
112-
return std::tie(RealmGroup, Realm, Domain);
117+
return std::tie(RealmGroup, Realm, Domain, Device);
113118
}
114119

115120
friend bool operator ==(const TPDiskLayoutPosition& x, const TPDiskLayoutPosition& y) {
@@ -124,12 +129,13 @@ namespace NKikimr::NBsController {
124129
struct TScore {
125130
ui32 RealmInterlace = 0;
126131
ui32 DomainInterlace = 0;
132+
ui32 DeviceInterlace = 0;
127133
ui32 RealmGroupScatter = 0;
128134
ui32 RealmScatter = 0;
129135
ui32 DomainScatter = 0;
130136

131137
auto AsTuple() const {
132-
return std::make_tuple(RealmInterlace, DomainInterlace, RealmGroupScatter, RealmScatter, DomainScatter);
138+
return std::make_tuple(RealmInterlace, DomainInterlace, DeviceInterlace, RealmGroupScatter, RealmScatter, DomainScatter);
133139
}
134140

135141
bool BetterThan(const TScore& other) const {
@@ -141,12 +147,13 @@ namespace NKikimr::NBsController {
141147
}
142148

143149
static TScore Max() {
144-
return {::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>()};
150+
return {::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>()};
145151
}
146152

147153
TString ToString() const {
148154
return TStringBuilder() << "{RealmInterlace# " << RealmInterlace
149155
<< " DomainInterlace# " << DomainInterlace
156+
<< " DeviceInterlace# " << DeviceInterlace
150157
<< " RealmGroupScatter# " << RealmGroupScatter
151158
<< " RealmScatter# " << RealmScatter
152159
<< " DomainScatter# " << DomainScatter
@@ -168,6 +175,8 @@ namespace NKikimr::NBsController {
168175
TStackVec<THashMap<TEntityId, ui32>, 32> NumDisksPerDomain;
169176
THashMap<TEntityId, ui32> NumDisksPerDomainTotal;
170177

178+
THashMap<TEntityId, ui32> NumDisksPerDevice;
179+
171180
TGroupLayout(const TBlobStorageGroupInfo::TTopology& topology)
172181
: Topology(topology)
173182
, NumDisksInRealm(Topology.GetTotalFailRealmsNum())
@@ -187,6 +196,8 @@ namespace NKikimr::NBsController {
187196
NumDisksInDomain[domainIdx] += value;
188197
NumDisksPerDomain[domainIdx][pos.Domain] += value;
189198
NumDisksPerDomainTotal[pos.Domain] += value;
199+
200+
NumDisksPerDevice[pos.Device] += value;
190201
}
191202

192203
void AddDisk(const TPDiskLayoutPosition& pos, ui32 orderNumber) {
@@ -201,12 +212,18 @@ namespace NKikimr::NBsController {
201212
const TVDiskIdShort vdisk = Topology.GetVDiskId(orderNumber);
202213
const ui32 domainIdx = Topology.GetFailDomainOrderNumber(vdisk);
203214

215+
const auto& disksPerRealm = NumDisksPerRealm[vdisk.FailRealm][pos.Realm];
216+
const auto& disksPerDomain = NumDisksPerDomain[domainIdx][pos.Domain];
217+
218+
const ui32 disksOnDevice = NumDisksPerDevice[pos.Device];
219+
204220
return {
205-
.RealmInterlace = NumDisksPerRealmTotal[pos.Realm] - NumDisksPerRealm[vdisk.FailRealm][pos.Realm],
206-
.DomainInterlace = NumDisksPerDomainTotal[pos.Domain] - NumDisksPerDomain[domainIdx][pos.Domain],
221+
.RealmInterlace = NumDisksPerRealmTotal[pos.Realm] - disksPerRealm,
222+
.DomainInterlace = NumDisksPerDomainTotal[pos.Domain] - disksPerDomain,
223+
.DeviceInterlace = disksOnDevice,
207224
.RealmGroupScatter = NumDisks - NumDisksPerRealmGroup[pos.RealmGroup],
208-
.RealmScatter = NumDisksInRealm[vdisk.FailRealm] - NumDisksPerRealm[vdisk.FailRealm][pos.Realm],
209-
.DomainScatter = NumDisksInDomain[domainIdx] - NumDisksPerDomain[domainIdx][pos.Domain],
225+
.RealmScatter = NumDisksInRealm[vdisk.FailRealm] - disksPerRealm,
226+
.DomainScatter = NumDisksInDomain[domainIdx] - disksPerDomain,
210227
};
211228
}
212229

‎ydb/core/mind/bscontroller/group_mapper.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ namespace NKikimr::NBsController {
390390

391391
static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
392392
scope = x.Domain;
393-
return {x, x};
393+
return {{x.RealmGroup, x.Realm, x.Domain, TEntityId::Min()}, {x.RealmGroup, x.Realm, x.Domain, TEntityId::Max()}};
394394
}
395395
};
396396

@@ -400,7 +400,7 @@ namespace NKikimr::NBsController {
400400

401401
static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
402402
scope = x.Realm;
403-
return {{x.RealmGroup, x.Realm, TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max()}};
403+
return {{x.RealmGroup, x.Realm, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max(), TEntityId::Max()}};
404404
}
405405
};
406406

@@ -410,7 +410,7 @@ namespace NKikimr::NBsController {
410410

411411
static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
412412
scope = x.RealmGroup;
413-
return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max()}};
413+
return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max(), TEntityId::Max()}};
414414
}
415415
};
416416

‎ydb/core/mind/bscontroller/group_mapper_ut.cpp

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -592,10 +592,6 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
592592
TestBlock42(1);
593593
}
594594

595-
Y_UNIT_TEST(Block42_2disk) {
596-
TestBlock42(2);
597-
}
598-
599595
Y_UNIT_TEST(Mirror3dc) {
600596
TTestContext context(6, 3, 3, 3, 3);
601597
TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureMirror3dc));
@@ -613,6 +609,23 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
613609
context.CheckIfGroupsAreMappedCompact();
614610
}
615611

612+
Y_UNIT_TEST(Mirror3dc3Nodes) {
613+
// Each node has 3 disks.
614+
TTestContext context(
615+
{
616+
{1, 1, 1, 1, 3},
617+
{2, 1, 2, 1, 3},
618+
{3, 1, 3, 1, 3},
619+
}
620+
);
621+
622+
TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureMirror3dc, 3, 3, 1, 10, 20, 10, 256));
623+
context.PopulateGroupMapper(mapper, 9);
624+
625+
TGroupMapper::TGroupDefinition group;
626+
UNIT_ASSERT_UNEQUAL(0, context.AllocateGroup(mapper, group));
627+
}
628+
616629
Y_UNIT_TEST(NonUniformCluster) {
617630
std::vector<std::tuple<ui32, ui32, ui32, ui32, ui32>> disks;
618631
for (ui32 rack = 0, body = 0; rack < 12; ++rack) {
@@ -637,6 +650,39 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
637650
}
638651
}
639652

653+
Y_UNIT_TEST(InterlacedRacksWithoutInterlacedNodes) {
654+
TTestContext context(
655+
{
656+
{1, 1, 1, 1, 1}, // node 1
657+
{1, 1, 2, 2, 1},
658+
{1, 1, 3, 3, 2}, // node 3 has two disks
659+
{1, 1, 4, 4, 1},
660+
{1, 1, 5, 5, 1},
661+
{1, 1, 6, 6, 1},
662+
{1, 1, 2, 7, 1}, // node 7 is in the same rack as node 2
663+
{1, 1, 8, 8, 1},
664+
{1, 1, 3, 9, 1}, // node 9 is in the same rack as node 3
665+
}
666+
);
667+
668+
TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::Erasure4Plus2Block));
669+
context.PopulateGroupMapper(mapper, 8);
670+
671+
TGroupMapper::TGroupDefinition group;
672+
group.emplace_back(TVector<TVector<TPDiskId>>(8));
673+
auto& g = group[0];
674+
675+
for (int i = 0; i < 8; i++) {
676+
g[i].emplace_back(TPDiskId(i + 1, 1));
677+
}
678+
679+
context.SetGroup(1, group);
680+
681+
TGroupMapper::TGroupDefinition newGroup = context.ReallocateGroup(mapper, 1, {TPDiskId(8, 1)});
682+
683+
UNIT_ASSERT_EQUAL_C(TPDiskId(9, 1), newGroup[0][7][0], context.FormatGroup(newGroup));
684+
}
685+
640686
Y_UNIT_TEST(NonUniformCluster2) {
641687
std::vector<std::tuple<ui32, ui32, ui32, ui32, ui32>> disks;
642688
for (ui32 rack = 0, body = 0; rack < 12; ++rack) {

0 commit comments

Comments
 (0)