Skip to content

Fix distconf automatic box management (merge from main #15429) #15447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 7, 2025
Merged

Fix distconf automatic box management (merge from main #15429) #15447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 141 additions & 44 deletions ydb/core/mind/bscontroller/bsc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,76 @@ void TBlobStorageController::Handle(TEvents::TEvUndelivered::TPtr ev) {
}
}

bool TBlobStorageController::HostConfigEquals(const THostConfigInfo& left, const NKikimrBlobStorage::TDefineHostConfig& right) const {
if (left.Name != right.GetName()) {
return false;
}

THashMap<TStringBuf, const THostConfigInfo::TDriveInfo*> driveMap;
for (const auto& [key, info] : left.Drives) {
driveMap.emplace(key.Path, &info);
}

TMaybe<TString> defaultPDiskConfig;
if (right.HasDefaultHostPDiskConfig()) {
const bool success = right.GetDefaultHostPDiskConfig().SerializeToString(&defaultPDiskConfig.ConstructInPlace());
Y_ABORT_UNLESS(success);
}

auto checkDrive = [&](const auto& drive) {
const auto it = driveMap.find(drive.GetPath());
if (it == driveMap.end()) {
return false;
}

if (drive.GetType() != it->second->Type ||
drive.GetSharedWithOs() != it->second->SharedWithOs ||
drive.GetReadCentric() != it->second->ReadCentric ||
drive.GetKind() != it->second->Kind) {
return false;
}

TMaybe<TString> pdiskConfig;
if (drive.HasPDiskConfig()) {
const bool success = drive.GetPDiskConfig().SerializeToString(&pdiskConfig.ConstructInPlace());
Y_ABORT_UNLESS(success);
} else {
pdiskConfig = defaultPDiskConfig;
}

if (pdiskConfig != it->second->PDiskConfig) {
return false;
}

driveMap.erase(it);
return true;
};

for (const auto& drive : right.GetDrive()) {
if (!checkDrive(drive)) {
return false;
}
}

auto addDrives = [&](const auto& field, NKikimrBlobStorage::EPDiskType type) {
NKikimrBlobStorage::THostConfigDrive drive;
drive.SetType(type);
for (const auto& path : field) {
if (drive.SetPath(path); !checkDrive(drive)) {
return false;
}
}
return true;
};
if (!addDrives(right.GetRot(), NKikimrBlobStorage::EPDiskType::ROT) ||
!addDrives(right.GetSsd(), NKikimrBlobStorage::EPDiskType::SSD) ||
!addDrives(right.GetNvme(), NKikimrBlobStorage::EPDiskType::NVME)) {
return false;
}

return driveMap.empty();
}

void TBlobStorageController::ApplyStorageConfig(bool ignoreDistconf) {
if (!StorageConfig.HasBlobStorageConfig()) {
return;
Expand All @@ -211,74 +281,101 @@ void TBlobStorageController::ApplyStorageConfig(bool ignoreDistconf) {
return; // not expected to be managed by BSC
}

ui64 expectedBoxId;
std::optional<ui64> generation;
bool needToDefineBox = true;
if (!Boxes.empty()) {
const auto& [boxId, box] = *Boxes.begin();

THashSet<THostConfigId> unusedHostConfigs;
for (const auto& [hostConfigId, _] : HostConfigs) {
unusedHostConfigs.insert(hostConfigId);
expectedBoxId = boxId;
generation = box.Generation.GetOrElse(1);
needToDefineBox = false;

// put all existing hosts of a singular box into the set
THashSet<std::tuple<TString, ui32, ui64, TMaybe<ui32>>> hosts;
for (const auto& [key, value] : box.Hosts) {
hosts.emplace(key.Fqdn, key.IcPort, value.HostConfigId, value.EnforcedNodeId);
}
for (const auto& [_, host] : box.Hosts) {
if (!HostConfigs.contains(host.HostConfigId)) {
return;

// drop matching entries from the new set
for (const auto& host : bsConfig.GetDefineBox().GetHost()) {
const auto& resolved = HostRecords->GetHostId(host.GetEnforcedNodeId());
Y_ABORT_UNLESS(resolved);
const auto& [fqdn, port] = *resolved;

if (!hosts.erase(std::make_tuple(fqdn, port, host.GetHostConfigId(), Nothing()))) {
needToDefineBox = true;
break;
}
unusedHostConfigs.erase(host.HostConfigId);
}

if (!unusedHostConfigs.empty()) {
return;
if (!hosts.empty()) {
needToDefineBox = true;
}

generation = box.Generation.GetOrElse(0);
}

auto ev = std::make_unique<TEvBlobStorage::TEvControllerConfigRequest>();
auto& r = ev->Record;
auto *request = r.MutableRequest();
for (const auto& hostConfig : bsConfig.GetDefineHostConfig()) {
const auto it = HostConfigs.find(hostConfig.GetHostConfigId());
if (it != HostConfigs.end() && HostConfigEquals(it->second, hostConfig)) {
continue;
}

auto *cmd = request->AddCommand();
auto *defineHostConfig = cmd->MutableDefineHostConfig();
defineHostConfig->CopyFrom(hostConfig);
if (it != HostConfigs.end()) {
defineHostConfig->SetItemConfigGeneration(it->second.Generation.GetOrElse(1));
}
}

if (needToDefineBox) {
auto *cmd = request->AddCommand();
cmd->MutableDefineHostConfig()->CopyFrom(hostConfig);
}
auto *cmd = request->AddCommand();
auto *defineBox = cmd->MutableDefineBox();
defineBox->CopyFrom(bsConfig.GetDefineBox());
defineBox->SetBoxId(1);
for (auto& host : *defineBox->MutableHost()) {
const ui32 nodeId = host.GetEnforcedNodeId();
host.ClearEnforcedNodeId();
auto *key = host.MutableKey();
const auto& resolved = HostRecords->GetHostId(nodeId);
Y_ABORT_UNLESS(resolved);
const auto& [fqdn, port] = *resolved;
key->SetFqdn(fqdn);
key->SetIcPort(port);
}
if (generation) {
defineBox->SetItemConfigGeneration(*generation);
}

THashSet<THostConfigId> unusedHostConfigs;
for (const auto& [hostConfigId, _] : HostConfigs) {
unusedHostConfigs.insert(hostConfigId);
}
for (const auto& host : defineBox->GetHost()) {
unusedHostConfigs.erase(host.GetHostConfigId());
}
for (const THostConfigId hostConfigId : unusedHostConfigs) {
auto *defineBox = cmd->MutableDefineBox();
defineBox->CopyFrom(bsConfig.GetDefineBox());
defineBox->SetBoxId(expectedBoxId);
for (auto& host : *defineBox->MutableHost()) {
const ui32 nodeId = host.GetEnforcedNodeId();
host.ClearEnforcedNodeId();
auto *key = host.MutableKey();
const auto& resolved = HostRecords->GetHostId(nodeId);
Y_ABORT_UNLESS(resolved);
const auto& [fqdn, port] = *resolved;
key->SetFqdn(fqdn);
key->SetIcPort(port);
}
if (generation) {
defineBox->SetItemConfigGeneration(*generation);
}
}

THashMap<THostConfigId, ui64> unusedHostConfigs;
for (const auto& [hostConfigId, value] : HostConfigs) {
unusedHostConfigs.emplace(hostConfigId, value.Generation.GetOrElse(1));
}
for (const auto& hostConfig : bsConfig.GetDefineHostConfig()) {
unusedHostConfigs.erase(hostConfig.GetHostConfigId());
}
for (const auto& [hostConfigId, generation] : unusedHostConfigs) {
auto *cmd = request->AddCommand();
auto *del = cmd->MutableDeleteHostConfig();
del->SetHostConfigId(hostConfigId);
del->SetItemConfigGeneration(HostConfigs[hostConfigId].Generation.GetOrElse(0));
del->SetItemConfigGeneration(generation);
}

STLOG(PRI_DEBUG, BS_CONTROLLER, BSC14, "ApplyStorageConfig", (Request, r));

Send(SelfId(), ev.release());
if (request->CommandSize()) {
STLOG(PRI_DEBUG, BS_CONTROLLER, BSC14, "ApplyStorageConfig", (Request, r));
Send(SelfId(), ev.release());
}
}

void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr ev) {
STLOG(PRI_DEBUG, BS_CONTROLLER, BSC15, "TEvControllerConfigResponse", (Response, ev->Get()->Record));
auto& record = ev->Get()->Record;
auto& response = record.GetResponse();
STLOG(response.GetSuccess() ? PRI_DEBUG : PRI_ERROR, BS_CONTROLLER, BSC15, "TEvControllerConfigResponse",
(Response, response));
}

void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerUpdateGroupStat::TPtr& ev) {
Expand Down
1 change: 1 addition & 0 deletions ydb/core/mind/bscontroller/impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1790,6 +1790,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa

void Handle(TEvNodeWardenStorageConfig::TPtr ev);
void Handle(TEvents::TEvUndelivered::TPtr ev);
bool HostConfigEquals(const THostConfigInfo& left, const NKikimrBlobStorage::TDefineHostConfig& right) const;
void ApplyStorageConfig(bool ignoreDistconf = false);
void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr ev);

Expand Down
Loading