Skip to content

maybe skip a balancer if it did nothing last time #14868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 21, 2025
Merged

maybe skip a balancer if it did nothing last time #14868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 37 additions & 22 deletions ydb/core/mind/hive/hive_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2383,6 +2383,17 @@ void THive::Handle(TEvPrivate::TEvProcessTabletBalancer::TPtr&) {
nodeUsageHistogram.IncrementFor(record.Usage * 100);
}

std::optional<TBalancerSettings> settings;
const auto maybeStartBalancer = [&settings, this]() -> bool {
Y_DEBUG_ABORT_UNLESS(settings);
if (LastBalancerTrigger == settings->Type
&& BalancerStats[static_cast<size_t>(settings->Type)].LastRunMovements == 0) {
return false;
}
StartHiveBalancer(std::move(*settings));
return true;
};

double minUsageToKick = GetMaxNodeUsageToKick() - GetNodeUsageRangeToKick();
if (stats.MaxUsage >= GetMaxNodeUsageToKick() && stats.MinUsage < minUsageToKick) {
std::vector<TNodeId> overloadedNodes;
Expand All @@ -2393,15 +2404,17 @@ void THive::Handle(TEvPrivate::TEvProcessTabletBalancer::TPtr&) {
}

if (!overloadedNodes.empty()) {
BLOG_D("Nodes " << overloadedNodes << " with usage over limit " << GetMaxNodeUsageToKick() << " - starting balancer");
StartHiveBalancer({
BLOG_D("Nodes " << overloadedNodes << " with usage over limit " << GetMaxNodeUsageToKick() << " - triggered balancer");
settings.emplace(TBalancerSettings{
.Type = EBalancerType::Emergency,
.MaxMovements = (int)CurrentConfig.GetMaxMovementsOnEmergencyBalancer(),
.RecheckOnFinish = CurrentConfig.GetContinueEmergencyBalancer(),
.MaxInFlight = GetEmergencyBalancerInflight(),
.FilterNodeIds = std::move(overloadedNodes),
});
return;
if (maybeStartBalancer()) {
return;
}
}
}

Expand All @@ -2410,24 +2423,19 @@ void THive::Handle(TEvPrivate::TEvProcessTabletBalancer::TPtr&) {
}

if (ObjectDistributions.GetMaxImbalance() > GetObjectImbalanceToBalance()) {
TInstant now = TActivationContext::Now();
if (LastBalancerTrigger != EBalancerType::SpreadNeighbours
|| BalancerStats[static_cast<std::size_t>(EBalancerType::SpreadNeighbours)].LastRunMovements != 0
|| BalancerStats[static_cast<std::size_t>(EBalancerType::SpreadNeighbours)].LastRunTimestamp + TDuration::Seconds(1) < now) {
auto objectToBalance = ObjectDistributions.GetObjectToBalance();
BLOG_D("Max imbalance " << ObjectDistributions.GetMaxImbalance() << " - starting balancer for object " << objectToBalance.ObjectId);
StartHiveBalancer({
.Type = EBalancerType::SpreadNeighbours,
.MaxMovements = (int)CurrentConfig.GetMaxMovementsOnAutoBalancer(),
.RecheckOnFinish = CurrentConfig.GetContinueAutoBalancer(),
.MaxInFlight = GetBalancerInflight(),
.FilterNodeIds = std::move(objectToBalance.Nodes),
.ResourceToBalance = EResourceToBalance::Counter,
.FilterObjectId = objectToBalance.ObjectId,
});
auto objectToBalance = ObjectDistributions.GetObjectToBalance();
BLOG_D("Max imbalance " << ObjectDistributions.GetMaxImbalance() << " - triggered balancer for object " << objectToBalance.ObjectId);
settings.emplace(TBalancerSettings{
.Type = EBalancerType::SpreadNeighbours,
.MaxMovements = (int)CurrentConfig.GetMaxMovementsOnAutoBalancer(),
.RecheckOnFinish = CurrentConfig.GetContinueAutoBalancer(),
.MaxInFlight = GetBalancerInflight(),
.FilterNodeIds = std::move(objectToBalance.Nodes),
.ResourceToBalance = EResourceToBalance::Counter,
.FilterObjectId = objectToBalance.ObjectId,
});
if (maybeStartBalancer()) {
return;
} else {
BLOG_D("Skipping SpreadNeigbours Balancer, now: " << now << ", allowed: " << BalancerStats[static_cast<std::size_t>(EBalancerType::SpreadNeighbours)].LastRunTimestamp + TDuration::Seconds(1));
}
}

Expand All @@ -2452,14 +2460,21 @@ void THive::Handle(TEvPrivate::TEvProcessTabletBalancer::TPtr&) {
break;
}
BLOG_TRACE("Scatter " << stats.ScatterByResource << " over limit "
<< GetMinScatterToBalance() << " - starting balancer " << EBalancerTypeName(balancerType));
StartHiveBalancer({
<< GetMinScatterToBalance() << " - triggered balancer " << EBalancerTypeName(balancerType));
settings.emplace(TBalancerSettings{
.Type = balancerType,
.MaxMovements = (int)CurrentConfig.GetMaxMovementsOnAutoBalancer(),
.RecheckOnFinish = CurrentConfig.GetContinueAutoBalancer(),
.MaxInFlight = GetBalancerInflight(),
.ResourceToBalance = *scatteredResource,
});
if (maybeStartBalancer()) {
return;
}
}

if (settings) {
StartHiveBalancer(std::move(*settings));
return;
}

Expand Down
108 changes: 91 additions & 17 deletions ydb/core/mind/hive/hive_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <ydb/core/testlib/basics/helpers.h>
#include <ydb/core/testlib/tablet_helpers.h>
#include <ydb/core/testlib/tenant_runtime.h>
#include <ydb/core/tx/columnshard/columnshard.h>
#include <ydb/core/tx/schemeshard/schemeshard.h>
#include <ydb/core/tx/mediator/mediator.h>
#include <ydb/core/util/random.h>
Expand Down Expand Up @@ -92,6 +93,7 @@ namespace {
runtime.SetLogPriority(NKikimrServices::TABLET_RESOLVER, otherPriority);
runtime.SetLogPriority(NKikimrServices::STATESTORAGE, otherPriority);
runtime.SetLogPriority(NKikimrServices::BOOTSTRAPPER, otherPriority);
runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, otherPriority);
}

THashMap<ui32, TIntrusivePtr<TNodeWardenConfig>> NodeWardenConfigs;
Expand Down Expand Up @@ -215,25 +217,34 @@ namespace {
false);
}

TLocalConfig::TPtr MakeDefaultLocalConfig() {
TLocalConfig::TPtr localConfig(new TLocalConfig());
localConfig->TabletClassInfo[TTabletTypes::Dummy].SetupInfo = new TTabletSetupInfo(
&CreateFlatDummyTablet,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
localConfig->TabletClassInfo[TTabletTypes::Hive].SetupInfo = new TTabletSetupInfo(
&CreateDefaultHive,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
localConfig->TabletClassInfo[TTabletTypes::Mediator].SetupInfo = new TTabletSetupInfo(
&CreateTxMediator,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
localConfig->TabletClassInfo[TTabletTypes::ColumnShard].SetupInfo = new TTabletSetupInfo(
&CreateColumnShard,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
return localConfig;
}

void SetupLocals(TTestActorRuntime &runtime, bool isLocalEnabled) {
if (!isLocalEnabled) {
return;
}

for (ui32 nodeIndex = 0; nodeIndex < runtime.GetNodeCount(); ++nodeIndex) {
TLocalConfig::TPtr localConfig(new TLocalConfig());
localConfig->TabletClassInfo[TTabletTypes::Dummy].SetupInfo = new TTabletSetupInfo(
&CreateFlatDummyTablet,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
localConfig->TabletClassInfo[TTabletTypes::Hive].SetupInfo = new TTabletSetupInfo(
&CreateDefaultHive,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
localConfig->TabletClassInfo[TTabletTypes::Mediator].SetupInfo = new TTabletSetupInfo(
&CreateTxMediator,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
auto localConfig = MakeDefaultLocalConfig();
TTenantPoolConfig::TPtr tenantPoolConfig = new TTenantPoolConfig(localConfig);
tenantPoolConfig->AddStaticSlot(DOMAIN_NAME);

Expand Down Expand Up @@ -653,10 +664,7 @@ Y_UNIT_TEST_SUITE(THiveTest) {

void CreateLocal(TTestActorRuntime &runtime, ui32 nodeIndex, TLocalConfig::TPtr localConfig = {}) {
if (localConfig == nullptr) {
localConfig = new TLocalConfig();
localConfig->TabletClassInfo[TTabletTypes::Dummy].SetupInfo = new TTabletSetupInfo(&CreateFlatDummyTablet,
TMailboxType::Simple, 0,
TMailboxType::Simple, 0);
localConfig = MakeDefaultLocalConfig();
}
TTenantPoolConfig::TPtr tenantPoolConfig = new TTenantPoolConfig(localConfig);
tenantPoolConfig->AddStaticSlot(DOMAIN_NAME);
Expand Down Expand Up @@ -5183,6 +5191,72 @@ Y_UNIT_TEST_SUITE(THiveTest) {
UNIT_ASSERT_EQUAL(initialDistribution, newDistribution);
}

Y_UNIT_TEST(TestHiveBalancerHighUsageAndColumnShards) {
static constexpr ui64 NUM_NODES = 2;
TTestBasicRuntime runtime(2, false);
Setup(runtime, true, 1, [](TAppPrepare& app) {
app.HiveConfig.SetTabletKickCooldownPeriod(0);
app.HiveConfig.SetResourceChangeReactionPeriod(0);
});
const int nodeBase = runtime.GetNodeId(0);
TActorId senderA = runtime.AllocateEdgeActor();
const ui64 hiveTablet = MakeDefaultHiveID();
const ui64 testerTablet = MakeTabletID(false, 1);

auto getDistribution = [hiveTablet, nodeBase, senderA, &runtime]() -> std::array<std::vector<ui64>, NUM_NODES> {
std::array<std::vector<ui64>, NUM_NODES> nodeTablets = {};
{
runtime.SendToPipe(hiveTablet, senderA, new TEvHive::TEvRequestHiveInfo());
TAutoPtr<IEventHandle> handle;
TEvHive::TEvResponseHiveInfo* response = runtime.GrabEdgeEventRethrow<TEvHive::TEvResponseHiveInfo>(handle);
for (const NKikimrHive::TTabletInfo& tablet : response->Record.GetTablets()) {
UNIT_ASSERT_C(((int)tablet.GetNodeID() - nodeBase >= 0) && (tablet.GetNodeID() - nodeBase < NUM_NODES),
"nodeId# " << tablet.GetNodeID() << " nodeBase# " << nodeBase);
nodeTablets[tablet.GetNodeID() - nodeBase].push_back(tablet.GetTabletID());
}
}
return nodeTablets;
};

CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive);

// wait for creation of nodes
{
TDispatchOptions options;
options.FinalEvents.emplace_back(TEvLocal::EvStatus, NUM_NODES);
runtime.DispatchEvents(options);
}
SendKillLocal(runtime, 1);

TTabletTypes::EType tabletType = TTabletTypes::ColumnShard;
for (size_t i = 0; i < 2; ++i) {
THolder<TEvHive::TEvCreateTablet> ev(new TEvHive::TEvCreateTablet(testerTablet, 100500 + i, tabletType, BINDED_CHANNELS));
ev->Record.SetObjectId(i);
ui64 tabletId = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(ev), 0, true);
MakeSureTabletIsUp(runtime, tabletId, 0);
}

{
TActorId sender = runtime.AllocateEdgeActor(0);
THolder<TEvHive::TEvTabletMetrics> metrics = MakeHolder<TEvHive::TEvTabletMetrics>();
metrics->Record.SetTotalNodeUsage(.95);

runtime.SendToPipe(hiveTablet, sender, metrics.Release(), 0);
}
CreateLocal(runtime, 1);

{
TDispatchOptions options;
options.FinalEvents.emplace_back(NHive::TEvPrivate::EvBalancerOut, 2);
runtime.DispatchEvents(options, TDuration::Seconds(10));
}

// Check that balancer moved a tablet
auto newDistribution = getDistribution();

UNIT_ASSERT_VALUES_EQUAL(newDistribution[0].size(), newDistribution[1].size());
}

Y_UNIT_TEST(TestUpdateTabletsObjectUpdatesMetrics) {
TTestBasicRuntime runtime(1, false);
Setup(runtime, true);
Expand Down
Loading