actually delete nodes if possible (#3120)

ydb/core/mind/hive/hive_events.h

@@ -31,6 +31,7 @@ struct TEvPrivate {
         EvRestartCancelled,
         EvProcessStorageBalancer,
         EvStorageBalancerOut,
+        EvDeleteNode,
         EvEnd
     };
 
@@ -110,6 +111,12 @@ struct TEvPrivate {
     struct TEvProcessStorageBalancer : TEventLocal<TEvProcessStorageBalancer, EvProcessStorageBalancer> {};
 
     struct TEvStorageBalancerOut : TEventLocal<TEvStorageBalancerOut, EvStorageBalancerOut> {};
+
+    struct TEvDeleteNode : TEventLocal<TEvDeleteNode, EvDeleteNode> {
+        TNodeId NodeId;
+
+        TEvDeleteNode(TNodeId nodeId) : NodeId(nodeId) {}
+    };
 };
 
 } // NHive

ydb/core/mind/hive/hive_impl.cpp

@@ -94,6 +94,18 @@ void THive::RestartPipeTx(ui64 tabletId) {
     }
 }
 
+bool THive::TryToDeleteNode(TNodeInfo* node) {
+    if (node->CanBeDeleted()) {
+        DeleteNode(node->Id);
+        return true;
+    }
+    if (!node->DeletionScheduled) {
+        Schedule(GetNodeDeletePeriod(), new TEvPrivate::TEvDeleteNode(node->Id));
+        node->DeletionScheduled = true;
+    }
+    return false;
+}
+
 void THive::Handle(TEvTabletPipe::TEvServerConnected::TPtr& ev) {
     if (ev->Get()->TabletId == TabletID()) {
         BLOG_TRACE("Handle TEvTabletPipe::TEvServerConnected(" << ev->Get()->ClientId << ") " << ev->Get()->ServerId);
@@ -108,9 +120,9 @@ void THive::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev) {
         TNodeInfo* node = FindNode(ev->Get()->ClientId.NodeId());
         if (node != nullptr) {
             Erase(node->PipeServers, ev->Get()->ServerId);
-            if (node->PipeServers.empty() && node->IsUnknown() && node->CanBeDeleted()) {
+            if (node->PipeServers.empty() && node->IsUnknown()) {
                 ObjectDistributions.RemoveNode(*node);
-                DeleteNode(node->Id);
+                TryToDeleteNode(node);
             }
         }
     }
@@ -2929,6 +2941,7 @@ void THive::ProcessEvent(std::unique_ptr<IEventHandle> event) {
         hFunc(TEvPrivate::TEvStartStorageBalancer, Handle);
         hFunc(TEvPrivate::TEvProcessStorageBalancer, Handle);
         hFunc(TEvHive::TEvUpdateDomain, Handle);
+        hFunc(TEvPrivate::TEvDeleteNode, Handle);
     }
 }
 
@@ -3029,6 +3042,7 @@ STFUNC(THive::StateWork) {
         fFunc(TEvPrivate::TEvStartStorageBalancer::EventType, EnqueueIncomingEvent);
         fFunc(TEvHive::TEvUpdateDomain::EventType, EnqueueIncomingEvent);
         fFunc(TEvPrivate::TEvProcessStorageBalancer::EventType, EnqueueIncomingEvent);
+        fFunc(TEvPrivate::TEvDeleteNode::EventType, EnqueueIncomingEvent);
         hFunc(TEvPrivate::TEvProcessIncomingEvent, Handle);
     default:
         if (!HandleDefaultEvents(ev, SelfId())) {
@@ -3293,6 +3307,17 @@ void THive::Handle(TEvPrivate::TEvLogTabletMoves::TPtr&) {
     TabletMovesByTypeForLog.clear();
 }
 
+void THive::Handle(TEvPrivate::TEvDeleteNode::TPtr& ev) {
+    auto node = FindNode(ev->Get()->NodeId);
+    if (node == nullptr) {
+        return;
+    }
+    node->DeletionScheduled = false;
+    if (!node->IsAlive()) {
+        TryToDeleteNode(node);
+    }
+}
+
 TVector<TNodeId> THive::GetNodesForWhiteboardBroadcast(size_t maxNodesToReturn) {
     TVector<TNodeId> nodes;
     TNodeId selfNodeId = SelfId().NodeId();

ydb/core/mind/hive/hive_impl.h

@@ -562,9 +562,11 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
     void Handle(TEvPrivate::TEvProcessStorageBalancer::TPtr& ev);
     void Handle(TEvPrivate::TEvProcessIncomingEvent::TPtr& ev);
     void Handle(TEvHive::TEvUpdateDomain::TPtr& ev);
+    void Handle(TEvPrivate::TEvDeleteNode::TPtr& ev);
 
 protected:
     void RestartPipeTx(ui64 tabletId);
+    bool TryToDeleteNode(TNodeInfo* node);
 
 public:
     static constexpr NKikimrServices::TActivity::EType ActorActivityType() {

ydb/core/mind/hive/node_info.h

@@ -61,6 +61,7 @@ struct TNodeInfo {
     mutable TInstant LastResourceChangeReaction;
     NKikimrHive::TNodeStatistics Statistics;
     TString Name;
+    bool DeletionScheduled = false;
 
     TNodeInfo(TNodeId nodeId, THive& hive);
     TNodeInfo(const TNodeInfo&) = delete;

ydb/core/mind/hive/tx__kill_node.cpp

@@ -52,9 +52,8 @@ class TTxKillNode : public TTransactionBase<THive> {
             }
             node->PipeServers.clear();
             Self->ObjectDistributions.RemoveNode(*node);
-            if (node->CanBeDeleted()) {
+            if (Self->TryToDeleteNode(node)) {
                 db.Table<Schema::Node>().Key(NodeId).Delete();
-                Self->DeleteNode(NodeId);
             } else {
                 db.Table<Schema::Node>().Key(NodeId).Update<Schema::Node::Local>(TActorId());
             }