Merge 5b747a0 into 4daa198

Author: alexvru

ydb/core/mind/bscontroller/bsc.cpp

@@ -111,6 +111,103 @@ void TBlobStorageController::TGroupInfo::CalculateLayoutStatus(TBlobStorageContr
     }
 }
 
+void TBlobStorageController::TGroupInfo::FillInGroupParameters(
+        NKikimrBlobStorage::TEvControllerSelectGroupsResult::TGroupParameters *params,
+        TBlobStorageController *self) const {
+    if (GroupMetrics) {
+        params->MergeFrom(GroupMetrics->GetGroupParameters());
+    } else if (BridgeGroupInfo) {
+        for (const auto& groupId : BridgeGroupInfo->GetBridgeGroupIds()) {
+            if (TGroupInfo *groupInfo = self->FindGroup(TGroupId::FromValue(groupId))) {
+                if (self->BridgeInfo && groupInfo->BridgePileId &&
+                        self->BridgeInfo->GetPile(*groupInfo->BridgePileId)->State == NKikimrBridge::TClusterState::DISCONNECTED) {
+                    continue; // ignore groups from disconnected piles
+                }
+                groupInfo->FillInGroupParameters(params, self);
+            } else {
+                Y_DEBUG_ABORT();
+            }
+        }
+    } else {
+        FillInResources(params->MutableAssuredResources(), true);
+        FillInResources(params->MutableCurrentResources(), false);
+        FillInVDiskResources(params);
+    }
+}
+
+void TBlobStorageController::TGroupInfo::FillInResources(
+        NKikimrBlobStorage::TEvControllerSelectGroupsResult::TGroupParameters::TResources *pb, bool countMaxSlots) const {
+    // count minimum params for each of slots assuming they are shared fairly between all the slots (expected or currently created)
+    std::optional<ui64> size;
+    std::optional<double> iops;
+    std::optional<ui64> readThroughput;
+    std::optional<ui64> writeThroughput;
+    std::optional<double> occupancy;
+    for (const TVSlotInfo *vslot : VDisksInGroup) {
+        const TPDiskInfo *pdisk = vslot->PDisk;
+        const auto& metrics = pdisk->Metrics;
+        const ui32 shareFactor = countMaxSlots ? pdisk->ExpectedSlotCount : pdisk->NumActiveSlots;
+        ui64 vdiskSlotSize = 0;
+        if (metrics.HasEnforcedDynamicSlotSize()) {
+            vdiskSlotSize = metrics.GetEnforcedDynamicSlotSize();
+        } else if (metrics.GetTotalSize()) {
+            vdiskSlotSize = metrics.GetTotalSize() / shareFactor;
+        }
+        if (vdiskSlotSize) {
+            size = Min(size.value_or(Max<ui64>()), vdiskSlotSize);
+        }
+        if (metrics.HasMaxIOPS()) {
+            iops = Min(iops.value_or(Max<double>()), metrics.GetMaxIOPS() * 100 / shareFactor * 0.01);
+        }
+        if (metrics.HasMaxReadThroughput()) {
+            readThroughput = Min(readThroughput.value_or(Max<ui64>()), metrics.GetMaxReadThroughput() / shareFactor);
+        }
+        if (metrics.HasMaxWriteThroughput()) {
+            writeThroughput = Min(writeThroughput.value_or(Max<ui64>()), metrics.GetMaxWriteThroughput() / shareFactor);
+        }
+        if (const auto& vm = vslot->Metrics; vm.HasOccupancy()) {
+            occupancy = Max(occupancy.value_or(0), vm.GetOccupancy());
+        }
+    }
+
+    // and recalculate it to the total size of the group according to the erasure
+    TBlobStorageGroupType type(ErasureSpecies);
+    const double factor = (double)VDisksInGroup.size() * type.DataParts() / type.TotalPartCount();
+    if (size) {
+        pb->SetSpace(Min<ui64>(pb->HasSpace() ? pb->GetSpace() : Max<ui64>(), *size * factor));
+    }
+    if (iops) {
+        pb->SetIOPS(Min<double>(pb->HasIOPS() ? pb->GetIOPS() : Max<double>(), *iops * VDisksInGroup.size() / type.TotalPartCount()));
+    }
+    if (readThroughput) {
+        pb->SetReadThroughput(Min<ui64>(pb->HasReadThroughput() ? pb->GetReadThroughput() : Max<ui64>(), *readThroughput * factor));
+    }
+    if (writeThroughput) {
+        pb->SetWriteThroughput(Min<ui64>(pb->HasWriteThroughput() ? pb->GetReadThroughput() : Max<ui64>(), *writeThroughput * factor));
+    }
+    if (occupancy) {
+        pb->SetOccupancy(Max<double>(pb->HasOccupancy() ? pb->GetOccupancy() : Min<double>(), *occupancy));
+    }
+}
+
+void TBlobStorageController::TGroupInfo::FillInVDiskResources(
+        NKikimrBlobStorage::TEvControllerSelectGroupsResult::TGroupParameters *pb) const {
+    TBlobStorageGroupType type(ErasureSpecies);
+    const double f = (double)VDisksInGroup.size() * type.DataParts() / type.TotalPartCount();
+    for (const TVSlotInfo *vslot : VDisksInGroup) {
+        const auto& m = vslot->Metrics;
+        if (m.HasAvailableSize()) {
+            pb->SetAvailableSize(Min<ui64>(pb->HasAvailableSize() ? pb->GetAvailableSize() : Max<ui64>(), f * m.GetAvailableSize()));
+        }
+        if (m.HasAllocatedSize()) {
+            pb->SetAllocatedSize(Max<ui64>(pb->HasAllocatedSize() ? pb->GetAllocatedSize() : 0, f * m.GetAllocatedSize()));
+        }
+        if (m.HasSpaceColor()) {
+            pb->SetSpaceColor(pb->HasSpaceColor() ? Max(pb->GetSpaceColor(), m.GetSpaceColor()) : m.GetSpaceColor());
+        }
+    }
+}
+
 NKikimrBlobStorage::TGroupStatus::E TBlobStorageController::DeriveStatus(const TBlobStorageGroupInfo::TTopology *topology,
         const TBlobStorageGroupInfo::TGroupVDisks& failed) {
     auto& checker = *topology->QuorumChecker;
@@ -189,7 +286,9 @@ void TBlobStorageController::Handle(TEvNodeWardenStorageConfig::TPtr ev) {
                 StaticVDiskMap.emplace(TVDiskID(vdiskId.GroupID, 0, vdiskId), vslotId);
                 ++StaticPDisks.at(pdiskId).StaticSlotUsage;
                 SysViewChangedVSlots.insert(vslotId);
-                SysViewChangedGroups.insert(vdiskId.GroupID);
+            }
+            for (const auto& group : ss.GetGroups()) {
+                SysViewChangedGroups.insert(TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID));
             }
         } else {
             Y_FAIL("no storage configuration provided");

ydb/core/mind/bscontroller/cmds_storage_pool.cpp

@@ -169,6 +169,8 @@ namespace NKikimr::NBsController {
                     GroupContentChanged.insert(it->second);
                 }
             }
+            // retain some fields
+            storagePool.BridgeMode = cur.BridgeMode;
             cur = std::move(storagePool); // update existing storage pool
         } else {
             // enable bridge mode by default for new pools (when bridge mode is enabled cluster-wide)

ydb/core/mind/bscontroller/config.cpp

@@ -712,6 +712,9 @@ namespace NKikimr::NBsController {
             }
             for (const auto& [base, overlay] : state.Groups.Diff()) {
                 SysViewChangedGroups.insert(overlay->first);
+                if (overlay->second && overlay->second->BridgeProxyGroupId) {
+                    SysViewChangedGroups.insert(*overlay->second->BridgeProxyGroupId);
+                }
             }
             for (const auto& [prev, cur] : Diff(&StoragePools, &state.StoragePools.Get())) {
                 SysViewChangedStoragePools.insert(cur ? cur->first : prev->first);
@@ -1109,10 +1112,7 @@ namespace NKikimr::NBsController {
             pb->SetExpectedStatus(status.ExpectedStatus);
 
             if (group.BridgeGroupInfo) {
-                NKikimrBlobStorage::TGroupInfo groupInfoPb;
-                bool success = groupInfoPb.ParseFromString(*group.BridgeGroupInfo);
-                Y_DEBUG_ABORT_UNLESS(success);
-                pb->SetIsProxyGroup(groupInfoPb.BridgeGroupIdsSize() != 0);
+                pb->SetIsProxyGroup(group.BridgeGroupInfo->BridgeGroupIdsSize() != 0);
             }
 
             if (group.DecommitStatus != NKikimrBlobStorage::TGroupDecommitStatus::NONE || group.VirtualGroupState) {
@@ -1241,8 +1241,7 @@ namespace NKikimr::NBsController {
             group->SetGroupSizeInUnits(groupInfo.GroupSizeInUnits);
 
             if (groupInfo.BridgeGroupInfo) {
-                const bool success = group->MergeFromString(*groupInfo.BridgeGroupInfo);
-                Y_DEBUG_ABORT_UNLESS(success);
+                group->MergeFrom(*groupInfo.BridgeGroupInfo);
             }
         }
 

ydb/core/mind/bscontroller/config_fit_groups.cpp

@@ -108,6 +108,7 @@ namespace NKikimr {
                         false, /* down */
                         false, /* seenOperational */
                         0, /* groupSizeInUnits */
+                        std::nullopt, /* bridgePileId */
                         StoragePoolId, /* storagePoolId */
                         0, /* numFailRealms */
                         0, /* numFailDomainsPerFailRealm */
@@ -120,22 +121,19 @@ namespace NKikimr {
                     auto& index = State.IndexGroupSpeciesToGroup.Unshare();
                     index[species].push_back(mainGroupId);
 
-                    NKikimrBlobStorage::TGroupInfo mainGroup;
+                    NKikimrBlobStorage::TGroupInfo& mainGroup = groupInfo->BridgeGroupInfo.emplace();
                     const auto& bridgeInfo = State.BridgeInfo;
                     Y_ABORT_UNLESS(bridgeInfo);
                     bridgeInfo->ForEachPile([&](TBridgePileId bridgePileId) {
-                        const TGroupId groupId = CreateGroup(bridgePileId);
+                        const TGroupId groupId = CreateGroup(bridgePileId, mainGroupId);
                         groupId.CopyToProto(&mainGroup, &NKikimrBlobStorage::TGroupInfo::AddBridgeGroupIds);
                     });
-
-                    const bool success = mainGroup.SerializeToString(&groupInfo->BridgeGroupInfo.ConstructInPlace());
-                    Y_DEBUG_ABORT_UNLESS(success);
                 } else {
-                    CreateGroup(std::nullopt); // regular single group
+                    CreateGroup(std::nullopt, std::nullopt); // regular single group
                 }
             }
 
-            TGroupId CreateGroup(std::optional<TBridgePileId> bridgePileId) {
+            TGroupId CreateGroup(std::optional<TBridgePileId> bridgePileId, std::optional<TGroupId> bridgeProxyGroupId) {
                 ////////////////////////////////////////////////////////////////////////////////////////////
                 // ALLOCATE GROUP ID FOR THE NEW GROUP
                 ////////////////////////////////////////////////////////////////////////////////////////////
@@ -187,12 +185,10 @@ namespace NKikimr {
                 TGroupInfo *groupInfo = State.Groups.ConstructInplaceNewEntry(groupId, groupId, 1,
                     0, Geometry.GetErasure(), desiredPDiskCategory.GetOrElse(0), StoragePool.VDiskKind,
                     StoragePool.EncryptionMode.GetOrElse(0), lifeCyclePhase, mainKeyId, encryptedGroupKey,
-                    groupKeyNonce, MainKeyVersion, false, false, groupSizeInUnits, StoragePoolId, Geometry.GetNumFailRealms(),
-                    Geometry.GetNumFailDomainsPerFailRealm(), Geometry.GetNumVDisksPerFailDomain());
+                    groupKeyNonce, MainKeyVersion, false, false, groupSizeInUnits, bridgePileId, StoragePoolId,
+                    Geometry.GetNumFailRealms(), Geometry.GetNumFailDomainsPerFailRealm(), Geometry.GetNumVDisksPerFailDomain());
 
-                if (bridgePileId) {
-                    groupInfo->BridgePileId = *bridgePileId;
-                }
+                groupInfo->BridgeProxyGroupId = bridgeProxyGroupId;
 
                 // bind group to storage pool
                 State.StoragePoolGroups.Unshare().emplace(StoragePoolId, groupId);
@@ -216,10 +212,6 @@ namespace NKikimr {
                     throw TExFitGroupError() << "GroupId# " << groupId << " not found";
                 }
 
-                std::optional<TBridgePileId> bridgePileId = groupInfo->BridgePileId
-                    ? std::make_optional(*groupInfo->BridgePileId)
-                    : std::nullopt;
-
                 TGroupMapper::TGroupDefinition group;
                 TGroupMapper::TGroupConstraintsDefinition softConstraints, hardConstraints;
                 bool layoutIsValid = true;
@@ -376,7 +368,7 @@ namespace NKikimr {
                             STLOG(PRI_INFO, BS_CONTROLLER, BSCFG01, "Attempt to sanitize group layout", (GroupId, groupId));
                             // Use group layout sanitizing algorithm on direct requests or when initial group layout is invalid
                             auto result = AllocateOrSanitizeGroup(groupId, group, {}, std::move(forbid), groupSizeInUnits, requiredSpace,
-                                AllowUnusableDisks, bridgePileId, &TGroupGeometryInfo::SanitizeGroup);
+                                AllowUnusableDisks, groupInfo->BridgePileId, &TGroupGeometryInfo::SanitizeGroup);
 
                             if (replacedSlots.empty()) {
                                 // update information about replaced disks
@@ -398,10 +390,10 @@ namespace NKikimr {
                             try {
                                 TGroupMapper::MergeTargetDiskConstraints(hardConstraints, softConstraints);
                                 AllocateOrSanitizeGroup(groupId, group, softConstraints, replacedDisks, std::move(forbid), groupSizeInUnits, requiredSpace,
-                                    AllowUnusableDisks, bridgePileId, &TGroupGeometryInfo::AllocateGroup);
+                                    AllowUnusableDisks, groupInfo->BridgePileId, &TGroupGeometryInfo::AllocateGroup);
                             } catch (const TExFitGroupError& ex) {
                                 AllocateOrSanitizeGroup(groupId, group, hardConstraints, replacedDisks, std::move(forbid), groupSizeInUnits, requiredSpace,
-                                    AllowUnusableDisks, bridgePileId, &TGroupGeometryInfo::AllocateGroup);
+                                    AllowUnusableDisks, groupInfo->BridgePileId, &TGroupGeometryInfo::AllocateGroup);
                             }
                         }
                         if (!IgnoreVSlotQuotaCheck) {
@@ -797,7 +789,9 @@ namespace NKikimr {
 
                     enumerateGroups([&](TGroupId groupId) {
                         fitter.CheckExistingGroup(groupId);
-                        ++numActualGroups;
+                        if (const TGroupInfo *group = state.Groups.Find(groupId); group && !group->BridgePileId) {
+                            ++numActualGroups;
+                        }
                     });
                     if (createNewGroups) {
                         if (numActualGroups < storagePool.NumGroups) {

ydb/core/mind/bscontroller/group_metrics_exchange.cpp

@@ -40,7 +40,7 @@ namespace NKikimr::NBsController {
                     if (TGroupInfo *group = Self->FindGroup(TGroupId::FromValue(groupId))) {
                         auto *item = outRecord.AddGroupMetrics();
                         item->SetGroupId(group->ID.GetRawId());
-                        group->FillInGroupParameters(item->MutableGroupParameters());
+                        group->FillInGroupParameters(item->MutableGroupParameters(), Self);
                     }
                 }
             }