improve failsafe on cuttting history

ydb/core/mind/hive/hive_impl.cpp

@@ -631,11 +631,8 @@ void THive::BuildLocalConfig() {
 }
 
 void THive::BuildCurrentConfig() {
-    BLOG_D("THive::BuildCurrentConfig ClusterConfig = " << ClusterConfig.ShortDebugString());
     CurrentConfig = ClusterConfig;
-    BLOG_D("THive::BuildCurrentConfig DatabaseConfig = " << DatabaseConfig.ShortDebugString());
     CurrentConfig.MergeFrom(DatabaseConfig);
-    BLOG_D("THive::BuildCurrentConfig CurrentConfig = " << CurrentConfig.ShortDebugString());
     TabletLimit.clear();
     for (const auto& tabletLimit : CurrentConfig.GetDefaultTabletLimit()) {
         TabletLimit.insert_or_assign(tabletLimit.GetType(), tabletLimit);
@@ -652,6 +649,22 @@ void THive::BuildCurrentConfig() {
         }
     }
     MakeTabletTypeSet(BalancerIgnoreTabletTypes);
+    CutHistoryDenyList.clear();
+    for (auto name : SplitString(CurrentConfig.GetCutHistoryDenyList(), ",")) {
+        TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name));
+        if (IsValidTabletType(type)) {
+            CutHistoryDenyList.emplace_back(type);
+        }
+    }
+    MakeTabletTypeSet(CutHistoryDenyList);
+    CutHistoryAllowList.clear();
+    for (auto name : SplitString(CurrentConfig.GetCutHistoryAllowList(), ",")) {
+        TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name));
+        if (IsValidTabletType(type)) {
+            CutHistoryAllowList.emplace_back(type);
+        }
+    }
+    MakeTabletTypeSet(CutHistoryAllowList);
     if (!CurrentConfig.GetSpreadNeighbours()) {
         // SpreadNeighbours can be turned off anytime, but
         // cannot be safely turned on without Hive restart
@@ -2223,9 +2236,8 @@ void THive::Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev)
     const NKikimrConsole::TConfigNotificationRequest& record = ev->Get()->Record;
     ClusterConfig = record.GetConfig().GetHiveConfig();
     NodeBrokerEpoch = TDuration::MicroSeconds(record.GetConfig().GetNodeBrokerConfig().GetEpochDuration());
-    BLOG_D("Received TEvConsole::TEvConfigNotificationRequest with update of cluster config: " << ClusterConfig.ShortDebugString() 
-           << "; " << record.GetConfig().GetNodeBrokerConfig().ShortDebugString());
     BuildCurrentConfig();
+    BLOG_D("Merged config: " << CurrentConfig);
     Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie);
 }
 

ydb/core/mind/hive/hive_impl.h

@@ -453,6 +453,8 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
 
     // normalized to be sorted list of unique values
     std::vector<TTabletTypes::EType> BalancerIgnoreTabletTypes; // built from CurrentConfig
+    std::vector<TTabletTypes::EType> CutHistoryDenyList; // built from CurrentConfig
+    std::vector<TTabletTypes::EType> CutHistoryAllowList; // built from CurrentConfig
 
     struct TTabletMoveInfo {
         TInstant Timestamp;
@@ -902,6 +904,21 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
         return (found != ignoreList.end());
     }
 
+    bool IsCutHistoryAllowed(TTabletTypes::EType type) const {
+        bool allowed = true;
+        const auto& denyList = CutHistoryDenyList;
+        if (!denyList.empty()) {
+            bool found = std::find(denyList.begin(), denyList.end(), type) != denyList.end();
+            allowed &= !found;
+        }
+        const auto& allowList = CutHistoryAllowList;
+        if (!allowList.empty()) {
+            bool found = std::find(allowList.begin(), allowList.end(), type) != allowList.end();
+            allowed &= found;
+        }
+        return allowed;
+    }
+
     double GetSpaceUsagePenaltyThreshold() {
         return CurrentConfig.GetSpaceUsagePenaltyThreshold();
     }

ydb/core/mind/hive/hive_impl_ut.cpp

@@ -218,3 +218,78 @@ Y_UNIT_TEST_SUITE(THiveImplTest) {
         UNIT_ASSERT_VALUES_EQUAL(stDev1, stDev2);
     }
 }
+
+Y_UNIT_TEST_SUITE(TCutHistoryRestrictions) {
+    class TTestHive : public THive {
+    public:
+        TTestHive(TTabletStorageInfo *info, const TActorId &tablet) : THive(info, tablet) {}
+
+        template<typename F>
+        void UpdateConfig(F func) {
+            func(ClusterConfig);
+            BuildCurrentConfig();
+        }
+    };
+
+    Y_UNIT_TEST(BasicTest) {
+        TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
+        hiveStorage->TabletType = TTabletTypes::Hive;
+        TTestHive hive(hiveStorage.Get(), TActorId());
+        hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
+            config.SetCutHistoryAllowList("DataShard,Coordinator");
+            config.SetCutHistoryDenyList("GraphShard");
+        });
+        UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard));
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::Hive));
+    }
+
+    Y_UNIT_TEST(EmptyAllowList) {
+        TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
+        hiveStorage->TabletType = TTabletTypes::Hive;
+        TTestHive hive(hiveStorage.Get(), TActorId());
+        hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
+            config.SetCutHistoryAllowList("");
+            config.SetCutHistoryDenyList("GraphShard");
+        });
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard));
+        UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::Hive));
+    }
+
+    Y_UNIT_TEST(EmptyDenyList) {
+        TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
+        hiveStorage->TabletType = TTabletTypes::Hive;
+        TTestHive hive(hiveStorage.Get(), TActorId());
+        hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
+            config.SetCutHistoryAllowList("DataShard,Coordinator");
+            config.SetCutHistoryDenyList("");
+        });
+        UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard));
+    }
+
+    Y_UNIT_TEST(SameTabletInBothLists) {
+        TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
+        hiveStorage->TabletType = TTabletTypes::Hive;
+        TTestHive hive(hiveStorage.Get(), TActorId());
+        hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
+            config.SetCutHistoryAllowList("DataShard,Coordinator");
+            config.SetCutHistoryDenyList("SchemeShard,DataShard");
+        });
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::SchemeShard));
+        UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::Hive));
+        UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::Coordinator));
+    }
+
+    Y_UNIT_TEST(BothListsEmpty) {
+        TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
+        hiveStorage->TabletType = TTabletTypes::Hive;
+        TTestHive hive(hiveStorage.Get(), TActorId());
+        hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
+            config.SetCutHistoryAllowList("");
+            config.SetCutHistoryDenyList("");
+        });
+        UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
+    }
+}

ydb/core/mind/hive/hive_schema.h

@@ -177,9 +177,10 @@ struct Schema : NIceDb::Schema {
         struct Group : Column<3, NScheme::NTypeIds::Uint64> {};
         struct Version : Column<4, NScheme::NTypeIds::Uint64> {};
         struct Timestamp : Column<5, NScheme::NTypeIds::Uint64> {};
+        struct DeletedAtGeneration : Column<6, NScheme::NTypeIds::Uint64> { static constexpr uint64_t Default = 0; };
 
         using TKey = TableKey<Tablet, Channel, Generation>;
-        using TColumns = TableColumns<Tablet, Channel, Generation, Group, Version, Timestamp>;
+        using TColumns = TableColumns<Tablet, Channel, Generation, Group, Version, Timestamp, DeletedAtGeneration>;
     };
 
     struct Node : Table<4> {

ydb/core/mind/hive/leader_tablet_info.cpp

@@ -367,12 +367,16 @@ void TLeaderTabletInfo::ActualizeTabletStatistics(TInstant now) {
     }
 }
 
-void TLeaderTabletInfo::RestoreDeletedHistory() {
-    for (const auto& entry : DeletedHistory) {
+void TLeaderTabletInfo::RestoreDeletedHistory(TTransactionContext& txc) {
+    NIceDb::TNiceDb db(txc.DB);
+    while (!DeletedHistory.empty()) {
+        const auto& entry = DeletedHistory.front();
         if (entry.Channel >= TabletStorageInfo->Channels.size()) {
             continue;
         }
         TabletStorageInfo->Channels[entry.Channel].History.push_back(entry.Entry);
+        db.Table<Schema::TabletChannelGen>().Key(Id, entry.Channel, entry.Entry.FromGeneration).Update<Schema::TabletChannelGen::DeletedAtGeneration>(0);
+        DeletedHistory.pop();
     }
 
     for (auto& channel : TabletStorageInfo->Channels) {
@@ -381,8 +385,6 @@ void TLeaderTabletInfo::RestoreDeletedHistory() {
             return lhs.FromGeneration < rhs.FromGeneration;
         });
     }
-
-    DeletedHistory.clear();
 }
 
 void TLeaderTabletInfo::SetType(TTabletTypes::EType type) {

ydb/core/mind/hive/leader_tablet_info.h

@@ -53,10 +53,12 @@ struct TLeaderTabletInfo : TTabletInfo {
     struct TChannelHistoryEntry {
         ui32 Channel;
         TTabletChannelInfo::THistoryEntry Entry;
+        ui32 DeletedAtGeneration;
 
-        TChannelHistoryEntry(ui32 channel, const TTabletChannelInfo::THistoryEntry& entry)
+        TChannelHistoryEntry(ui32 channel, const TTabletChannelInfo::THistoryEntry& entry, ui32 deletedAtGeneration)
             : Channel(channel)
             , Entry(entry)
+            , DeletedAtGeneration(deletedAtGeneration)
         {
         }
     };
@@ -71,8 +73,8 @@ struct TLeaderTabletInfo : TTabletInfo {
     TIntrusivePtr<TTabletStorageInfo> TabletStorageInfo;
     TChannelsBindings BoundChannels;
     std::bitset<MAX_TABLET_CHANNELS> ChannelProfileNewGroup;
-    std::vector<TChannelHistoryEntry> DeletedHistory;
-    bool WasAliveSinceCutHistory = false;
+    std::queue<TChannelHistoryEntry> DeletedHistory;
+    bool WasAliveSinceCutHistory = true;
     NKikimrHive::TEvReassignTablet::EHiveReassignReason ChannelProfileReassignReason;
     ui32 KnownGeneration;
     TTabletCategoryInfo* Category;
@@ -352,7 +354,7 @@ struct TLeaderTabletInfo : TTabletInfo {
     TString GetChannelStoragePoolName(const TChannelProfiles::TProfile::TChannel& channel) const;
     TString GetChannelStoragePoolName(ui32 channelId) const;
     TStoragePoolInfo& GetStoragePool(ui32 channelId) const;
-    void RestoreDeletedHistory();
+    void RestoreDeletedHistory(TTransactionContext& txc);
 
     void SetType(TTabletTypes::EType type);
 };

ydb/core/mind/hive/tx__cut_tablet_history.cpp

@@ -14,12 +14,12 @@ class TTxCutTabletHistory : public TTransactionBase<THive> {
 
     TTxType GetTxType() const override { return NHive::TXTYPE_CUT_TABLET_HISTORY; }
 
-    bool Execute(TTransactionContext&, const TActorContext&) override {
+    bool Execute(TTransactionContext& txc, const TActorContext&) override {
         TEvHive::TEvCutTabletHistory* msg = Event->Get();
         auto tabletId = msg->Record.GetTabletID();
         BLOG_D("THive::TTxCutTabletHistory::Execute(" << tabletId << ")");
         TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(tabletId);
-        if (tablet != nullptr && tablet->IsReadyToReassignTablet()) {
+        if (tablet != nullptr && tablet->IsReadyToReassignTablet() && Self->IsCutHistoryAllowed(tablet->Type)) {
             auto channel = msg->Record.GetChannel();
             Y_ABORT_UNLESS(channel < tablet->TabletStorageInfo->Channels.size());
             TTabletChannelInfo& channelInfo = tablet->TabletStorageInfo->Channels[channel];
@@ -30,11 +30,11 @@ class TTxCutTabletHistory : public TTransactionBase<THive> {
                         channelInfo.History.end(),
                         TTabletChannelInfo::THistoryEntry(fromGeneration, groupId));
             if (it != channelInfo.History.end()) {
-                tablet->DeletedHistory.emplace_back(channel, *it);
+                Self->TabletCounters->Cumulative()[NHive::COUNTER_HISTORY_CUT].Increment(1);
+                tablet->DeletedHistory.emplace(channel, *it, tablet->KnownGeneration);
                 channelInfo.History.erase(it);
-                /* to be safe, don't do it just yet
                 NIceDb::TNiceDb db(txc.DB);
-                db.Table<Schema::TabletChannelGen>().Key(tabletId, channel, fromGeneration).Delete();*/
+                db.Table<Schema::TabletChannelGen>().Key(tabletId, channel, fromGeneration).Update<Schema::TabletChannelGen::DeletedAtGeneration>(tablet->KnownGeneration);
             }
         }
         return true;

ydb/core/mind/hive/tx__load_everything.cpp

@@ -533,8 +533,14 @@ class TTxLoadEverything : public TTransactionBase<THive> {
                         tablet->TabletStorageInfo->Channels.emplace_back();
                         tablet->TabletStorageInfo->Channels.back().Channel = tablet->TabletStorageInfo->Channels.size() - 1;
                     }
-                    TTabletChannelInfo& channel = tablet->TabletStorageInfo->Channels[channelId];
-                    channel.History.emplace_back(generationId, groupId, timestamp);
+                    TTabletChannelInfo::THistoryEntry entry(generationId, groupId, timestamp);
+                    auto deletedAtGeneration = tabletChannelGenRowset.GetValueOrDefault<Schema::TabletChannelGen::DeletedAtGeneration>();
+                    if (deletedAtGeneration) {
+                        tablet->DeletedHistory.emplace(channelId, entry, deletedAtGeneration);
+                    } else {
+                        TTabletChannelInfo& channel = tablet->TabletStorageInfo->Channels[channelId];
+                        channel.History.push_back(entry);
+                    }
                 } else {
                     ++numMissingTablets;
                 }

ydb/core/mind/hive/tx__start_tablet.cpp

@@ -55,7 +55,8 @@ class TTxStartTablet : public TTransactionBase<THive> {
                     if (!leader.DeletedHistory.empty()) {
                         if (!leader.WasAliveSinceCutHistory) {
                             BLOG_ERROR("THive::TTxStartTablet::Execute Tablet " << TabletId << " failed to start after cutting history - will restore history");
-                            leader.RestoreDeletedHistory();
+                            Self->TabletCounters->Cumulative()[NHive::COUNTER_HISTORY_RESTORED].Increment(leader.DeletedHistory.size());
+                            leader.RestoreDeletedHistory(txc);
                         } else {
                             leader.WasAliveSinceCutHistory = false;
                         }

ydb/core/mind/hive/tx__update_tablet_status.cpp

@@ -124,6 +124,14 @@ class TTxUpdateTabletStatus : public TTransactionBase<THive> {
                                                                     NIceDb::TUpdate<Schema::Tablet::KnownGeneration>(Generation),
                                                                     NIceDb::TUpdate<Schema::Tablet::Statistics>(tablet->Statistics));
                     Self->UpdateTabletFollowersNumber(leader, db, SideEffects);
+
+                    // tablet booted successfully, we may actually cut history now
+                    while (!leader.DeletedHistory.empty() && leader.DeletedHistory.front().DeletedAtGeneration < leader.KnownGeneration) {
+                        leader.WasAliveSinceCutHistory = true;
+                        const auto& entry = leader.DeletedHistory.front();
+                        db.Table<Schema::TabletChannelGen>().Key(TabletId, entry.Channel, entry.Entry.FromGeneration).Delete();
+                        leader.DeletedHistory.pop();
+                    }
                 } else {
                     db.Table<Schema::TabletFollowerTablet>().Key(TabletId, FollowerId).Update(
                                 NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(tablet->AsFollower().FollowerGroup.Id),
@@ -160,13 +168,6 @@ class TTxUpdateTabletStatus : public TTransactionBase<THive> {
                             db.Table<Schema::Tablet>().Key(TabletId).Update(NIceDb::TUpdate<Schema::Tablet::LeaderNode>(0),
                                                                             NIceDb::TUpdate<Schema::Tablet::KnownGeneration>(leader.KnownGeneration),
                                                                             NIceDb::TUpdate<Schema::Tablet::Statistics>(tablet->Statistics));
-
-                            // tablet booted successfully, we may actually cut history now
-                            leader.WasAliveSinceCutHistory = true;
-                            for (const auto& entry : leader.DeletedHistory) {
-                                db.Table<Schema::TabletChannelGen>().Key(TabletId, entry.Channel, entry.Entry.FromGeneration).Delete();
-                            }
-                            leader.DeletedHistory.clear();
                         } else {
                             db.Table<Schema::TabletFollowerTablet>().Key(TabletId, FollowerId).Update(
                                         NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(tablet->AsFollower().FollowerGroup.Id),

ydb/core/protos/config.proto

@@ -1485,6 +1485,8 @@ message THiveConfig {
     optional double NodeUsageRangeToKick = 75 [default = 0.2];
     optional bool LessSystemTabletsMoves = 77 [default = true];
     optional uint64 MaxPingsInFlight = 78 [default = 1000];
+    optional string CutHistoryDenyList = 76 [default = "ColumnShard,KeyValue,PersQueue,BlobDepot"];
+    optional string CutHistoryAllowList = 79 [default = "DataShard"];
 }
 
 message TBlobCacheConfig {

ydb/core/protos/counters_hive.proto

@@ -48,6 +48,8 @@ enum ECumulativeCounters {
     COUNTER_SUGGESTED_SCALE_DOWN = 11 [(CounterOpts) = {Name: "SuggestedScaleDown"}];
     COUNTER_STORAGE_BALANCER_EXECUTED = 12 [(CounterOpts) = {Name: "StorageBalancerExecuted"}];
     COUNTER_TABLETS_STORAGE_REASSIGNED = 13 [(CounterOpts) = {Name: "TabletsStorageReassigned"}];
+    COUNTER_HISTORY_CUT = 14 [(CounterOpts) = {Name: "TabletHistoryCut"}];
+    COUNTER_HISTORY_RESTORED = 15 [(CounterOpts) = {Name: "TabletHistoryRestored"}];
 }
 
 enum EPercentileCounters {

ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema

@@ -520,6 +520,11 @@
                 "ColumnId": 5,
                 "ColumnName": "Timestamp",
                 "ColumnType": "Uint64"
+            },
+            {
+                "ColumnId": 6,
+                "ColumnName": "DeletedAtGeneration",
+                "ColumnType": "Uint64"
             }
         ],
         "ColumnsDropped": [],
@@ -531,7 +536,8 @@
                     2,
                     3,
                     4,
-                    5
+                    5,
+                    6
                 ],
                 "RoomID": 0,
                 "Codec": 0,