Skip to content

Commit ff307c7

Browse files
alexvrualexvru
authored
Support intersecting hosts in MergeBoxes (#12282)
1 parent 06b7c08 commit ff307c7

File tree

3 files changed

+168
-8
lines changed

3 files changed

+168
-8
lines changed

‎ydb/core/mind/bscontroller/cmds_box.cpp‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -108,17 +108,46 @@ namespace NKikimr::NBsController {
108108
TBoxInfo& origin = getBox(cmd.GetOriginBoxId(), cmd.GetOriginBoxGeneration(), "origin");
109109
TBoxInfo& target = getBox(cmd.GetTargetBoxId(), cmd.GetTargetBoxGeneration(), "target");
110110

111-
while (origin.Hosts) {
112-
auto node = origin.Hosts.extract(origin.Hosts.begin());
113-
node.key().BoxId = cmd.GetTargetBoxId();
114-
if (!target.Hosts.insert(std::move(node)).inserted) {
115-
throw TExError() << "duplicate hosts in merged box";
111+
THashMap<std::tuple<THostConfigId, THostConfigId>, THostConfigId> mergedHostConfigs;
112+
113+
for (auto& [originKey, originValue] : origin.Hosts) {
114+
const TBoxInfo::THostKey targetKey = originKey.ChangeBox(cmd.GetTargetBoxId()); // make new key for target set
115+
if (const auto [it, inserted] = target.Hosts.try_emplace(targetKey, std::move(originValue)); !inserted) {
116+
TBoxInfo::THostInfo& targetValue = it->second;
117+
if (targetValue.EnforcedNodeId != originValue.EnforcedNodeId) {
118+
throw TExError() << "different EnforcedNodeId for the same HostId# " << THostId(originKey);
119+
} else {
120+
const auto sourceConfigs = std::make_tuple(std::min(originValue.HostConfigId, targetValue.HostConfigId),
121+
std::max(originValue.HostConfigId, targetValue.HostConfigId));
122+
123+
const auto [jt, inserted] = mergedHostConfigs.try_emplace(sourceConfigs);
124+
if (inserted) {
125+
auto& hostConfigs = HostConfigs.Unshare();
126+
jt->second = hostConfigs.empty() ? 1 : std::prev(hostConfigs.end())->first + 1;
127+
Y_ABORT_UNLESS(!hostConfigs.contains(jt->second));
128+
THostConfigInfo& newHostConfig = hostConfigs[jt->second];
129+
newHostConfig.Generation = 1;
130+
131+
auto addDrivesFrom = [&](THostConfigId hostConfigId) {
132+
const auto it = hostConfigs.find(hostConfigId);
133+
if (it == hostConfigs.end()) {
134+
throw TExError() << "missing HostConfig for origin/target box";
135+
}
136+
for (const auto& [key, value] : it->second.Drives) {
137+
const THostConfigInfo::TDriveKey newKey{{jt->second, key.Path}};
138+
if (const auto [it, inserted] = newHostConfig.Drives.emplace(newKey, value); !inserted) {
139+
throw TExError() << "duplicate drives in origin/target host configs";
140+
}
141+
}
142+
};
143+
addDrivesFrom(originValue.HostConfigId);
144+
addDrivesFrom(targetValue.HostConfigId);
145+
}
146+
targetValue.HostConfigId = jt->second;
147+
}
116148
}
117149
}
118150

119-
// spin the generation
120-
target.Generation = target.Generation.GetOrElse(1) + 1;
121-
122151
auto& storagePools = StoragePools.Unshare();
123152
auto& storagePoolGroups = StoragePoolGroups.Unshare();
124153

‎ydb/core/mind/bscontroller/impl.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -996,6 +996,10 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
996996
std::tie(BoxId, Fqdn, IcPort) = std::move(key);
997997
}
998998

999+
THostKey ChangeBox(Schema::BoxHostV2::BoxId::Type newBoxId) const {
1000+
return {{newBoxId, Fqdn, IcPort}};
1001+
}
1002+
9991003
auto GetKey() const {
10001004
return std::tie(BoxId, Fqdn, IcPort);
10011005
}

‎ydb/core/mind/bscontroller/ut_bscontroller/main.cpp‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -714,6 +714,133 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) {
714714
});
715715
}
716716

717+
Y_UNIT_TEST(MergeIntersectingBoxes) {
718+
const ui32 numNodes = 50;
719+
const ui32 numNodes1 = 20;
720+
const ui32 numGroups1 = numNodes * 3;
721+
const ui32 numGroups2 = numNodes1 * 4;
722+
TEnvironmentSetup env(numNodes, 1);
723+
RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) {
724+
TFinalizer finalizer(env);
725+
env.Prepare(dispatchName, setup, outActiveZone);
726+
727+
TVector<TEnvironmentSetup::TNodeRecord> nodes1, nodes2;
728+
for (const auto& node : env.GetNodes()) {
729+
nodes1.push_back(node);
730+
if (nodes2.size() < numNodes1) {
731+
nodes2.push_back(node);
732+
}
733+
}
734+
735+
TSet<ui32> nodeIds1, nodeIds2;
736+
for (const auto& item : nodes1) {
737+
nodeIds1.insert(std::get<2>(item));
738+
}
739+
for (const auto& item : nodes2) {
740+
nodeIds2.insert(std::get<2>(item));
741+
}
742+
743+
NKikimrBlobStorage::TConfigRequest request;
744+
env.DefineBox(1, "first box", {
745+
{"/dev/disk1", NKikimrBlobStorage::ROT, false, false, 0},
746+
{"/dev/disk2", NKikimrBlobStorage::ROT, true, false, 0},
747+
{"/dev/disk3", NKikimrBlobStorage::SSD, false, false, 0},
748+
}, nodes1, request);
749+
env.DefineStoragePool(1, 1, "first storage pool", numGroups1, NKikimrBlobStorage::ROT, {}, request);
750+
751+
NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
752+
UNIT_ASSERT(response.GetSuccess());
753+
754+
request.Clear();
755+
env.DefineBox(2, "second box", {
756+
{"/dev/disk4", NKikimrBlobStorage::ROT, false, false, 0},
757+
{"/dev/disk5", NKikimrBlobStorage::ROT, true, false, 0},
758+
{"/dev/disk6", NKikimrBlobStorage::SSD, false, false, 0},
759+
}, nodes2, request);
760+
env.DefineStoragePool(2, 1, "second storage pool", numGroups2, NKikimrBlobStorage::ROT, {}, request);
761+
762+
response = env.Invoke(request);
763+
UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
764+
765+
////////////////////////////////////////////////////////////////////////////////////////////////////////
766+
// merge boxes
767+
768+
request.Clear();
769+
auto& cmd = *request.AddCommand()->MutableMergeBoxes();
770+
cmd.SetOriginBoxId(2);
771+
cmd.SetOriginBoxGeneration(1);
772+
cmd.SetTargetBoxId(1);
773+
cmd.SetTargetBoxGeneration(1);
774+
auto& item = *cmd.AddStoragePoolIdMap();
775+
item.SetOriginStoragePoolId(1);
776+
item.SetTargetStoragePoolId(2);
777+
778+
response = env.Invoke(request);
779+
UNIT_ASSERT(response.GetSuccess());
780+
781+
////////////////////////////////////////////////////////////////////////////////////////////////////////
782+
// validate result
783+
784+
request.Clear();
785+
request.AddCommand()->MutableReadBox();
786+
request.AddCommand()->MutableReadHostConfig();
787+
request.AddCommand()->MutableQueryBaseConfig();
788+
response = env.Invoke(request);
789+
UNIT_ASSERT(response.GetSuccess());
790+
791+
THashMap<std::tuple<TString, i32>, ui32> nodeMap;
792+
for (const auto& node : env.GetNodes()) {
793+
nodeMap.emplace(std::make_tuple(std::get<0>(node), std::get<1>(node)), std::get<2>(node));
794+
}
795+
796+
// check box nodes
797+
using TDrives = std::set<TString>;
798+
THashMap<ui64, TDrives> hostConfigs;
799+
for (const auto& hostConfig : response.GetStatus(1).GetHostConfig()) {
800+
std::set<TString>& paths = hostConfigs[hostConfig.GetHostConfigId()];
801+
for (const auto& drive : hostConfig.GetDrive()) {
802+
const auto [it, inserted] = paths.insert(drive.GetPath());
803+
UNIT_ASSERT(inserted);
804+
}
805+
}
806+
const auto& boxes = response.GetStatus(0).GetBox();
807+
UNIT_ASSERT_VALUES_EQUAL(boxes.size(), 1);
808+
const auto& box = boxes.at(0);
809+
UNIT_ASSERT_VALUES_EQUAL(box.HostSize(), numNodes);
810+
for (const auto& item : box.GetHost()) {
811+
UNIT_ASSERT(hostConfigs.contains(item.GetHostConfigId()));
812+
const auto& hostConfig = hostConfigs[item.GetHostConfigId()];
813+
const auto& key = item.GetKey();
814+
const auto keyTuple = std::make_tuple(key.GetFqdn(), key.GetIcPort());
815+
UNIT_ASSERT(nodeMap.contains(keyTuple));
816+
const ui32 nodeId = nodeMap[keyTuple];
817+
if (nodeIds2.contains(nodeId)) { // 6 drives
818+
UNIT_ASSERT_EQUAL(hostConfig, (TDrives{"/dev/disk1", "/dev/disk2", "/dev/disk3", "/dev/disk4", "/dev/disk5", "/dev/disk6"}));
819+
} else { // 3 drives
820+
UNIT_ASSERT_EQUAL(hostConfig, (TDrives{"/dev/disk1", "/dev/disk2", "/dev/disk3"}));
821+
}
822+
}
823+
824+
// validate base config
825+
ui32 num1 = 0, num2 = 0;
826+
{
827+
const auto& baseConfig = response.GetStatus(2).GetBaseConfig();
828+
for (const auto& pdisk : baseConfig.GetPDisk()) {
829+
UNIT_ASSERT_VALUES_EQUAL(pdisk.GetBoxId(), 1);
830+
}
831+
for (const auto& group : baseConfig.GetGroup()) {
832+
UNIT_ASSERT_VALUES_EQUAL(group.GetBoxId(), 1);
833+
const ui64 storagePoolId = group.GetStoragePoolId();
834+
UNIT_ASSERT(storagePoolId == 1 || storagePoolId == 2);
835+
++(storagePoolId == 1 ? num1 : num2);
836+
}
837+
UNIT_ASSERT_VALUES_EQUAL(num1, numGroups1);
838+
UNIT_ASSERT_VALUES_EQUAL(num2, numGroups2);
839+
}
840+
841+
});
842+
}
843+
717844
Y_UNIT_TEST(MoveGroups) {
718845
const ui32 numNodes = 50;
719846
const ui32 numGroups1 = 100;

0 commit comments

Comments
 (0)