Bootstrapper: full node list in TBootstrapperInfo, handle possible duplicates (#9848)

Author: snaury

ydb/core/driver_lib/run/kikimr_services_initializers.cpp

@@ -1420,33 +1420,28 @@ void TBootstrapperInitializer::InitializeServices(
                     TActorSetupCmd(CreateConfiguredTabletBootstrapper(boot), TMailboxType::HTSwap, appData->SystemPoolId)));
             } else {
                 const bool standby = boot.HasStandBy() && boot.GetStandBy();
-                for (const ui32 bootstrapperNode : boot.GetNode()) {
-                    if (bootstrapperNode == NodeId) {
-
-                        TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(boot.GetInfo()));
-
-                        auto tabletType = BootstrapperTypeToTabletType(boot.GetType());
-
-                        auto tabletSetupInfo = CreateTablet(
-                            TTabletTypes::TypeToStr(tabletType),
-                            info,
-                            appData);
-
-                        TIntrusivePtr<TBootstrapperInfo> bi = new TBootstrapperInfo(tabletSetupInfo.Get());
-
-                        if (boot.NodeSize() != 1) {
-                            bi->OtherNodes.reserve(boot.NodeSize() - 1);
-                            for (ui32 x : boot.GetNode())
-                                if (x != NodeId)
-                                    bi->OtherNodes.push_back(x);
-                            if (boot.HasWatchThreshold())
-                                bi->WatchThreshold = TDuration::MilliSeconds(boot.GetWatchThreshold());
-                            if (boot.HasStartFollowers())
-                                bi->StartFollowers = boot.GetStartFollowers();
-                        }
-
-                        setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeBootstrapperID(info->TabletID, bootstrapperNode), TActorSetupCmd(CreateBootstrapper(info.Get(), bi.Get(), standby), TMailboxType::HTSwap, appData->SystemPoolId)));
-                    }
+                if (Find(boot.GetNode(), NodeId) != boot.GetNode().end()) {
+                    TIntrusivePtr<TTabletStorageInfo> info(TabletStorageInfoFromProto(boot.GetInfo()));
+
+                    auto tabletType = BootstrapperTypeToTabletType(boot.GetType());
+
+                    auto tabletSetupInfo = CreateTablet(
+                        TTabletTypes::TypeToStr(tabletType),
+                        info,
+                        appData);
+
+                    TIntrusivePtr<TBootstrapperInfo> bi = new TBootstrapperInfo(tabletSetupInfo.Get());
+                    bi->Nodes.reserve(boot.NodeSize());
+                    for (ui32 x : boot.GetNode())
+                        bi->Nodes.push_back(x);
+                    if (boot.HasWatchThreshold())
+                        bi->WatchThreshold = TDuration::MilliSeconds(boot.GetWatchThreshold());
+                    if (boot.HasStartFollowers())
+                        bi->StartFollowers = boot.GetStartFollowers();
+
+                    setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(
+                        MakeBootstrapperID(info->TabletID, NodeId),
+                        TActorSetupCmd(CreateBootstrapper(info.Get(), bi.Get(), standby), TMailboxType::HTSwap, appData->SystemPoolId)));
                 }
             }
         }

ydb/core/mind/configured_tablet_bootstrapper.cpp

@@ -82,16 +82,13 @@ class TConfiguredTabletBootstrapper : public TActorBootstrapped<TConfiguredTable
             TIntrusivePtr<TTabletSetupInfo> tabletSetupInfo = MakeTabletSetupInfo(tabletType, appData->UserPoolId, appData->SystemPoolId);
 
             TIntrusivePtr<TBootstrapperInfo> bi = new TBootstrapperInfo(tabletSetupInfo.Get());
-            if (config.NodeSize() != 1) {
-                for (ui32 node : config.GetNode()) {
-                    if (node != selfNode)
-                        bi->OtherNodes.emplace_back(node);
-                }
-                if (config.HasWatchThreshold())
-                    bi->WatchThreshold = TDuration::MilliSeconds(config.GetWatchThreshold());
-                if (config.HasStartFollowers())
-                    bi->StartFollowers = config.GetStartFollowers();
+            for (ui32 node : config.GetNode()) {
+                bi->Nodes.emplace_back(node);
             }
+            if (config.HasWatchThreshold())
+                bi->WatchThreshold = TDuration::MilliSeconds(config.GetWatchThreshold());
+            if (config.HasStartFollowers())
+                bi->StartFollowers = config.GetStartFollowers();
 
             BootstrapperInstance = Register(CreateBootstrapper(storageInfo.Get(), bi.Get(), false), TMailboxType::HTSwap, appData->SystemPoolId);
 

ydb/core/tablet/bootstrapper.cpp

@@ -20,6 +20,8 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
     const TIntrusivePtr<TTabletStorageInfo> TabletInfo;
     const TIntrusivePtr<TBootstrapperInfo> BootstrapperInfo;
     bool ModeStandby;
+    TVector<ui32> OtherNodes;
+    THashMap<ui32, size_t> OtherNodesIndex;
 
     TActorId KnownLeaderPipe;
 
@@ -63,10 +65,12 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
 
         TVector<TAlien> Aliens;
         TVector<TWatcher> Watchers;
+        size_t Waiting;
 
         explicit TRound(size_t count)
             : Aliens(count)
             , Watchers(count)
+            , Waiting(count)
         {}
     };
 
@@ -134,10 +138,22 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
         return NKikimrBootstrapper::TEvWatchResult::EState_Name(state).c_str();
     }
 
+    void BuildOtherNodes() {
+        ui32 selfNodeId = SelfId().NodeId();
+        for (ui32 nodeId : BootstrapperInfo->Nodes) {
+            if (nodeId != selfNodeId && !OtherNodesIndex.contains(nodeId)) {
+                size_t index = OtherNodes.size();
+                OtherNodes.push_back(nodeId);
+                OtherNodesIndex[nodeId] = index;
+            }
+        }
+    }
+
     size_t AlienIndex(ui32 alienNodeId) {
-        for (size_t i = 0, e = BootstrapperInfo->OtherNodes.size(); i != e; ++i)
-            if (BootstrapperInfo->OtherNodes[i] == alienNodeId)
-                return i;
+        auto it = OtherNodesIndex.find(alienNodeId);
+        if (it != OtherNodesIndex.end()) {
+            return it->second;
+        }
         return Max<size_t>();
     }
 
@@ -300,7 +316,7 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
         // Note: make sure notifications from previous states don't interfere
         ++RoundCounter;
 
-        if (BootstrapperInfo->OtherNodes.empty()) {
+        if (OtherNodes.empty()) {
             return Boot();
         }
 
@@ -311,8 +327,8 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
 
         const ui64 tabletId = TabletInfo->TabletID;
 
-        Round.emplace(BootstrapperInfo->OtherNodes.size());
-        for (ui32 alienNode : BootstrapperInfo->OtherNodes) {
+        Round.emplace(OtherNodes.size());
+        for (ui32 alienNode : OtherNodes) {
             Send(MakeBootstrapperID(tabletId, alienNode),
                 new TEvBootstrapper::TEvWatch(tabletId, SelfSeed, RoundCounter),
                 IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
@@ -407,6 +423,10 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
             "tablet: " << TabletInfo->TabletID << ", type: " << GetTabletTypeName()
             << ", apply alien " << alien.NodeId() << " state: " << GetStateName(state));
 
+        if (alienEntry.State == TRound::EAlienState::Wait) {
+            Y_ABORT_UNLESS(Round->Waiting-- > 0);
+        }
+
         alienEntry.Seed = seed;
 
         switch (state) {
@@ -434,17 +454,22 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
     }
 
     void CheckRoundCompletion() {
+        auto& round = Round.value();
+        if (round.Waiting > 0) {
+            return;
+        }
+
         ui64 winnerSeed = SelfSeed;
         ui32 winner = SelfId().NodeId();
 
         size_t undelivered = 0;
         size_t disconnected = 0;
-        auto& round = Round.value();
         for (size_t i = 0, e = round.Aliens.size(); i != e; ++i) {
             const auto& alien = round.Aliens[i];
-            const ui32 node = BootstrapperInfo->OtherNodes.at(i);
+            const ui32 node = OtherNodes.at(i);
             switch (alien.State) {
                 case TRound::EAlienState::Wait:
+                    Y_DEBUG_ABORT("Unexpected Wait state");
                     return;
                 case TRound::EAlienState::Unknown:
                     break;
@@ -485,7 +510,7 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
 
     bool CheckBootPermitted(size_t undelivered, size_t disconnected) {
         // Total number of nodes that participate in tablet booting
-        size_t total = 1 + BootstrapperInfo->OtherNodes.size();
+        size_t total = 1 + OtherNodes.size();
         Y_DEBUG_ABORT_UNLESS(total >= 1 + undelivered + disconnected);
 
         // Ignore nodes that don't have bootstrapper running
@@ -785,7 +810,7 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
     }
 
     void PassAway() override {
-        for (ui32 nodeId : BootstrapperInfo->OtherNodes) {
+        for (ui32 nodeId : OtherNodes) {
             Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe);
         }
         NotifyWatchers();
@@ -806,6 +831,7 @@ class TBootstrapper : public TActorBootstrapped<TBootstrapper> {
     }
 
     void Bootstrap() {
+        BuildOtherNodes();
         if (ModeStandby) {
             Become(&TThis::StateStandBy);
         } else {

ydb/core/tablet/bootstrapper.h

@@ -33,16 +33,13 @@ struct TEvBootstrapper {
 
 struct TBootstrapperInfo : public TThrRefBase {
     TIntrusivePtr<TTabletSetupInfo> SetupInfo;
-    TVector<ui32> OtherNodes;
-    TDuration WatchThreshold;
-    TDuration OfflineDelay;
-    bool StartFollowers;
+    TVector<ui32> Nodes;
+    TDuration WatchThreshold = TDuration::MilliSeconds(200);
+    TDuration OfflineDelay = TDuration::Seconds(3);
+    bool StartFollowers = false;
 
-    TBootstrapperInfo(TTabletSetupInfo* setupInfo)
+    explicit TBootstrapperInfo(TTabletSetupInfo* setupInfo)
         : SetupInfo(setupInfo)
-        , WatchThreshold(TDuration::MilliSeconds(200))
-        , OfflineDelay(TDuration::Seconds(3))
-        , StartFollowers(false)
     {}
 };
 

ydb/core/tablet/bootstrapper_ut.cpp

@@ -95,12 +95,16 @@ Y_UNIT_TEST_SUITE(BootstrapperTest) {
         std::vector<TActorId> boots;
         auto tabletInfo = CreateSimpleTabletStorageInfo(tabletId);
         auto setupInfo = CreateSimpleTabletSetupInfo();
+        auto bootInfo = MakeIntrusive<TBootstrapperInfo>(setupInfo.Get());
         for (ui32 nodeIdx : nodeIdxs) {
-            auto bootInfo = MakeIntrusive<TBootstrapperInfo>(setupInfo.Get());
-            for (ui32 otherNodeIdx : nodeIdxs) {
-                if (otherNodeIdx != nodeIdx) {
-                    bootInfo->OtherNodes.push_back(runtime.GetNodeId(otherNodeIdx));
-                }
+            bootInfo->Nodes.push_back(runtime.GetNodeId(nodeIdx));
+        }
+        THashMap<ui32, TActorId> started;
+        for (ui32 nodeIdx : nodeIdxs) {
+            if (started.contains(nodeIdx)) {
+                // Start one bootstrapper per node
+                boots.push_back(started.at(nodeIdx));
+                continue;
             }
             boots.push_back(runtime.Register(CreateBootstrapper(tabletInfo.Get(), bootInfo.Get()), nodeIdx));
             runtime.EnableScheduleForActor(boots.back());
@@ -109,6 +113,7 @@ Y_UNIT_TEST_SUITE(BootstrapperTest) {
                 MakeBootstrapperID(tabletId, runtime.GetNodeId(nodeIdx)),
                 boots.back(),
                 nodeIdx);
+            started[nodeIdx] = boots.back();
         }
         return boots;
     }
@@ -423,6 +428,24 @@ Y_UNIT_TEST_SUITE(BootstrapperTest) {
         Y_UNUSED(client2);
     }
 
+    Y_UNIT_TEST(DuplicateNodes) {
+        TTestBasicRuntime runtime(3);
+        SetupTabletServices(runtime);
+        runtime.SetLogPriority(NKikimrServices::BOOTSTRAPPER, NActors::NLog::PRI_DEBUG);
+
+        StartSimpleTabletBootstrappers(runtime, {1, 1, 2, 2});
+
+        size_t boots = 0;
+        auto observer = runtime.AddObserver<TEvTablet::TEvBoot>([&](auto&) {
+            ++boots;
+        });
+
+        runtime.SimulateSleep(TDuration::Seconds(1));
+
+        // Tablet must boot exactly once
+        UNIT_ASSERT_VALUES_EQUAL(boots, 1u);
+    }
+
 } // Y_UNIT_TEST_SUITE(BootstrapperTest)
 
 } // namespace NKikimr