Skip to content

Commit f895e52

Browse files
lex007inlex007in
authored
Tablet sys: Retry failed GC requests (#18228)
1 parent d2ef12e commit f895e52

File tree

6 files changed

+119
-24
lines changed

6 files changed

+119
-24
lines changed

‎ydb/core/tablet/tablet_impl.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ struct TEvTabletBase {
3030
EvFollowerRetry = EvBlockBlobStorageResult + 512,
3131
EvTrySyncFollower,
3232
EvTryBuildFollowerGraph,
33+
EvLogGcRetry,
3334

3435
EvEnd
3536
};
@@ -167,6 +168,8 @@ struct TEvTabletBase {
167168
, TabletId(tabletId)
168169
{}
169170
};
171+
172+
struct TEvLogGcRetry : public TEventLocal<TEvLogGcRetry, EvLogGcRetry> {};
170173
};
171174

172175
}

‎ydb/core/tablet/tablet_sys.cpp‎

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ namespace {
3535
static constexpr TDuration OfflineFollowerWaitFirst = TDuration::Seconds(4);
3636
static constexpr TDuration OfflineFollowerWaitRetry = TDuration::Seconds(15);
3737

38+
constexpr ui64 GcErrorInitialBackoffMs = 1;
39+
constexpr ui64 GcErrorMaxBackoffMs = 10000;
40+
constexpr ui64 GcMaxErrors = 25; // ~1.13 min in total
41+
3842
}
3943

4044
ui64 TTablet::TabletID() const {
@@ -1323,6 +1327,16 @@ void TTablet::Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr &ev) {
13231327

13241328
TEvBlobStorage::TEvCollectGarbageResult *msg = ev->Get();
13251329

1330+
auto handleNextGcLogChannel = [&]() {
1331+
if (GcNextStep != 0) {
1332+
GcLogChannel(std::exchange(GcNextStep, 0));
1333+
} else if (GcFailCount > 0 && !GcPendingRetry && GcTryCounter < GcMaxErrors) {
1334+
++GcTryCounter;
1335+
GcPendingRetry = true;
1336+
Schedule(TDuration::MilliSeconds(GcBackoffTimer.NextBackoffMs()), new TEvTabletBase::TEvLogGcRetry());
1337+
}
1338+
};
1339+
13261340
switch (msg->Status) {
13271341
case NKikimrProto::RACE:
13281342
case NKikimrProto::BLOCKED:
@@ -1335,18 +1349,27 @@ void TTablet::Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr &ev) {
13351349
}
13361350
break;
13371351
case NKikimrProto::OK:
1338-
if (GcInFly == 0 && GcForStepAckRequest) {
1339-
const auto& req = *GcForStepAckRequest->Get();
1340-
const ui32 gen = StateStorageInfo.KnownGeneration;
1341-
if (std::tie(req.Generation, req.Step) <= std::tie(gen, GcInFlyStep)) {
1342-
Send(GcForStepAckRequest->Sender, new TEvTablet::TEvGcForStepAckResponse(gen, GcInFlyStep));
1343-
GcForStepAckRequest = nullptr;
1352+
if (GcInFly == 0) {
1353+
if (GcFailCount == 0) {
1354+
GcConfirmedStep = GcInFlyStep;
1355+
GcTryCounter = 0;
1356+
GcBackoffTimer.Reset();
1357+
if (GcForStepAckRequest) {
1358+
const auto& req = *GcForStepAckRequest->Get();
1359+
const ui32 gen = StateStorageInfo.KnownGeneration;
1360+
if (std::tie(req.Generation, req.Step) <= std::tie(gen, GcConfirmedStep)) {
1361+
Send(GcForStepAckRequest->Sender, new TEvTablet::TEvGcForStepAckResponse(gen, GcConfirmedStep));
1362+
GcForStepAckRequest = nullptr;
1363+
}
1364+
}
13441365
}
1366+
handleNextGcLogChannel();
13451367
}
1346-
[[fallthrough]];
1347-
default: // silently ignore unrecognized errors (assume temporary)
1348-
if (GcInFly == 0 && GcNextStep != 0) {
1349-
GcLogChannel(std::exchange(GcNextStep, 0));
1368+
return;
1369+
default:
1370+
++GcFailCount;
1371+
if (GcInFly == 0) {
1372+
handleNextGcLogChannel();
13501373
}
13511374
return;
13521375
}
@@ -1357,8 +1380,8 @@ void TTablet::Handle(TEvBlobStorage::TEvCollectGarbageResult::TPtr &ev) {
13571380
void TTablet::Handle(TEvTablet::TEvGcForStepAckRequest::TPtr& ev) {
13581381
const auto& req = *ev->Get();
13591382
const ui32 gen = StateStorageInfo.KnownGeneration;
1360-
if (GcInFly == 0 && std::tie(req.Generation, req.Step) <= std::tie(gen, GcInFlyStep)) {
1361-
Send(ev->Sender, new TEvTablet::TEvGcForStepAckResponse(gen, GcInFlyStep));
1383+
if (GcInFly == 0 && std::tie(req.Generation, req.Step) <= std::tie(gen, GcConfirmedStep)) {
1384+
Send(ev->Sender, new TEvTablet::TEvGcForStepAckResponse(gen, GcConfirmedStep));
13621385
} else {
13631386
GcForStepAckRequest = ev;
13641387
}
@@ -1367,6 +1390,8 @@ void TTablet::Handle(TEvTablet::TEvGcForStepAckRequest::TPtr& ev) {
13671390
void TTablet::GcLogChannel(ui32 step) {
13681391
const ui64 tabletid = TabletID();
13691392
const ui32 gen = StateStorageInfo.KnownGeneration;
1393+
GcPendingRetry = false;
1394+
GcFailCount = 0;
13701395

13711396
if (GcInFly != 0 || Graph.SyncCommit.SyncStep != 0 && Graph.SyncCommit.SyncStep <= step) {
13721397
if (GcInFlyStep < step) {
@@ -1414,6 +1439,12 @@ void TTablet::GcLogChannel(ui32 step) {
14141439
GcNextStep = 0;
14151440
}
14161441

1442+
void TTablet::RetryGcRequests() {
1443+
if (GcPendingRetry && GcInFly == 0 && GcInFlyStep > GcConfirmedStep) {
1444+
GcLogChannel(GcInFlyStep);
1445+
}
1446+
}
1447+
14171448
void TTablet::SpreadFollowerAuxUpdate(const TString& auxUpdate) {
14181449
for (auto &xpair : LeaderInfo) {
14191450
SendFollowerAuxUpdate(xpair.second, xpair.first, auxUpdate);
@@ -1938,7 +1969,12 @@ TTablet::TTablet(const TActorId &launcher, TTabletStorageInfo *info, TTabletSetu
19381969
, DiscoveredLastBlocked(Max<ui32>())
19391970
, GcInFly(0)
19401971
, GcInFlyStep(0)
1972+
, GcConfirmedStep(0)
19411973
, GcNextStep(0)
1974+
, GcTryCounter(0)
1975+
, GcBackoffTimer(GcErrorInitialBackoffMs, GcErrorMaxBackoffMs)
1976+
, GcPendingRetry(false)
1977+
, GcFailCount(0)
19421978
, ResourceProfiles(profiles)
19431979
, TxCacheQuota(txCacheQuota)
19441980
{

‎ydb/core/tablet/tablet_sys.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <ydb/library/actors/core/interconnect.h>
66
#include <ydb/core/node_whiteboard/node_whiteboard.h>
77
#include <ydb/core/base/tablet_pipe.h>
8+
#include <ydb/core/util/backoff.h>
89
#include <ydb/library/actors/core/hfunc.h>
910
#include <util/generic/intrlist.h>
1011
#include <util/generic/set.h>
@@ -229,7 +230,15 @@ class TTablet : public TActor<TTablet> {
229230
ui32 DiscoveredLastBlocked;
230231
ui32 GcInFly;
231232
ui32 GcInFlyStep;
233+
ui32 GcConfirmedStep;
232234
ui32 GcNextStep;
235+
236+
// retry failed GC logic
237+
ui32 GcTryCounter;
238+
TBackoffTimer GcBackoffTimer;
239+
bool GcPendingRetry;
240+
ui32 GcFailCount;
241+
233242
TEvTablet::TEvGcForStepAckRequest::TPtr GcForStepAckRequest;
234243
TResourceProfilesPtr ResourceProfiles;
235244
TSharedQuotaPtr TxCacheQuota;
@@ -326,6 +335,7 @@ class TTablet : public TActor<TTablet> {
326335
void Handle(TEvTablet::TEvPreCommit::TPtr &ev);
327336

328337
void Handle(TEvTablet::TEvGcForStepAckRequest::TPtr& ev);
338+
void Handle(TEvTabletBase::TEvLogGcRetry::TPtr& ev);
329339

330340
void Handle(TEvTablet::TEvConfirmLeader::TPtr &ev);
331341
void Handle(TEvBlobStorage::TEvGetBlockResult::TPtr &ev);
@@ -342,6 +352,7 @@ class TTablet : public TActor<TTablet> {
342352
void DoSyncToFollower(TMap<TActorId, TLeaderInfo>::iterator followerIt);
343353

344354
void GcLogChannel(ui32 step);
355+
void RetryGcRequests();
345356

346357
bool ProgressCommitQueue();
347358
void ProgressFollowerQueue();
@@ -550,6 +561,7 @@ class TTablet : public TActor<TTablet> {
550561
hFunc(TEvBlobStorage::TEvCollectGarbageResult, Handle);
551562
hFunc(TEvBlobStorage::TEvGetBlockResult, Handle);
552563
hFunc(TEvTablet::TEvGcForStepAckRequest, Handle);
564+
sFunc(TEvTabletBase::TEvLogGcRetry, RetryGcRequests);
553565
}
554566
}
555567

‎ydb/core/tablet_flat/flat_executor_gclogic.cpp‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ namespace NKikimr {
66
namespace NTabletFlatExecutor {
77

88
namespace {
9-
constexpr ui64 GCErrorInitialBackoffMs = 20;
10-
constexpr ui64 GCErrorMaxBackoffMs = 10000;
11-
constexpr ui64 GCMaxErrors = 20; // ~3.33 min in total
9+
constexpr ui64 GcErrorInitialBackoffMs = 1;
10+
constexpr ui64 GcErrorMaxBackoffMs = 10000;
11+
constexpr ui64 GcMaxErrors = 25; // ~1.13 min in total
1212
}
1313

1414
TExecutorGCLogic::TExecutorGCLogic(TIntrusiveConstPtr<TTabletStorageInfo> info, TAutoPtr<NPageCollection::TSteppedCookieAllocator> cookies)
@@ -218,8 +218,8 @@ TExecutorGCLogic::TChannelInfo::TChannelInfo()
218218
: GcCounter(1)
219219
, GcWaitFor(0)
220220
, TryCounter(0)
221-
, BackoffTimer(GCErrorInitialBackoffMs, GCErrorMaxBackoffMs)
222-
, RetryIsScheduled(false)
221+
, BackoffTimer(GcErrorInitialBackoffMs, GcErrorMaxBackoffMs)
222+
, PendingRetry(false)
223223
, FailCount(0)
224224
{
225225
}
@@ -380,7 +380,7 @@ void TExecutorGCLogic::TChannelInfo::SendCollectGarbage(TGCTime uncommittedTime,
380380
return;
381381

382382
MinUncollectedTime = uncommittedTime;
383-
RetryIsScheduled = false;
383+
PendingRetry = false;
384384
FailCount = 0;
385385

386386
TVector<TLogoBlobID> keep;
@@ -494,17 +494,17 @@ void TExecutorGCLogic::TChannelInfo::OnCollectGarbageFailure() {
494494

495495
TDuration TExecutorGCLogic::TChannelInfo::TryScheduleGcRequestRetries() {
496496
if (GcWaitFor == 0 && FailCount > 0) {
497-
if (!RetryIsScheduled && TryCounter < GCMaxErrors) {
497+
if (!PendingRetry && TryCounter < GcMaxErrors) {
498498
++TryCounter;
499-
RetryIsScheduled = true;
499+
PendingRetry = true;
500500
return TDuration::MilliSeconds(BackoffTimer.NextBackoffMs());
501501
}
502502
}
503503
return TDuration{};
504504
}
505505

506506
void TExecutorGCLogic::TChannelInfo::RetryGcRequests(const TTabletStorageInfo *tabletStorageInfo, ui32 channel, ui32 generation, const TActorContext& ctx) {
507-
if (RetryIsScheduled) {
507+
if (PendingRetry) {
508508
SendCollectGarbage(MinUncollectedTime, tabletStorageInfo, channel, generation, ctx);
509509
}
510510
}

‎ydb/core/tablet_flat/flat_executor_gclogic.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ class TExecutorGCLogic {
9595
// retry failed GC logic
9696
ui32 TryCounter;
9797
TBackoffTimer BackoffTimer;
98-
bool RetryIsScheduled;
98+
bool PendingRetry;
9999
ui32 FailCount;
100100

101101
inline TChannelInfo();

‎ydb/core/tablet_flat/ut/ut_data_cleanup.cpp‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ enum : ui32 {
2626

2727
class TTxInitSchema : public ITransaction {
2828
public:
29-
TTxInitSchema(const TVector<ui32>& tableIds, bool addColumnsInFamily = false)
29+
TTxInitSchema(const TVector<ui32>& tableIds, bool addColumnsInFamily = false, bool allowLogBatching = false)
3030
: TableIds(tableIds)
3131
, AddColumnsInFamily(addColumnsInFamily)
32+
, AllowLogBatching(allowLogBatching)
3233
{ }
3334

3435
bool Execute(TTransactionContext& txc, const TActorContext&) override {
@@ -45,7 +46,8 @@ class TTxInitSchema : public ITransaction {
4546
.AddColumn(tableId, "key", KeyColumnId, NScheme::TInt64::TypeId, false)
4647
.AddColumn(tableId, "value", ValueColumnId, NScheme::TString::TypeId, false)
4748
.AddColumnToKey(tableId, KeyColumnId)
48-
.SetCompactionPolicy(tableId, policy);
49+
.SetCompactionPolicy(tableId, policy)
50+
.SetExecutorAllowLogBatching(AllowLogBatching);
4951

5052
if (AddColumnsInFamily) {
5153
txc.DB.Alter()
@@ -74,6 +76,7 @@ class TTxInitSchema : public ITransaction {
7476
private:
7577
const TVector<ui32> TableIds;
7678
const bool AddColumnsInFamily;
79+
const bool AllowLogBatching;
7780
};
7881

7982
class TTxWriteRow : public ITransaction {
@@ -694,12 +697,53 @@ Y_UNIT_TEST_SUITE(DataCleanup) {
694697
}
695698
gcResults.Stop().Unblock();
696699
retriedGcResults.Wait();
700+
retriedGcResults.Stop();
697701

698702
auto ev2 = env.GrabEdgeEvent<NFake::TEvDataCleaned>();
699703
UNIT_ASSERT_VALUES_EQUAL(ev2->Get()->DataCleanupGeneration, 235);
700704

701705
UNIT_ASSERT_VALUES_EQUAL(BlobStorageValueCountInAllGroups(env, value42), 0);
702706
UNIT_ASSERT_VALUES_EQUAL(BlobStorageValueCountInAllGroups(env, value43), 0);
703707
}
708+
709+
Y_UNIT_TEST(CleanupDataWithSysTabletGCErrors) {
710+
TString value42 = "Some_value";
711+
712+
TMyEnvBase env;
713+
env.FireDummyTablet(TestTabletFlags);
714+
env.SendSync(new NFake::TEvExecute{ new TTxInitSchema({ 101 }, false, true) });
715+
env.SendSync(new NFake::TEvExecute{ new TTxWriteRow(101, 42, value42) });
716+
717+
int readRows = 0;
718+
env.SendSync(new NFake::TEvExecute{ new TTxFullScan(101, readRows) }, true);
719+
UNIT_ASSERT_EQUAL(readRows, 1);
720+
721+
UNIT_ASSERT_VALUES_EQUAL(BlobStorageValueCount(env, value42, 0), 1);
722+
723+
// make all log GC responses failed
724+
auto obs = env->AddObserver<TEvBlobStorage::TEvCollectGarbageResult>([](auto& ev) {
725+
if (ev->Get()->Channel == 0) {
726+
ev->Get()->Status = NKikimrProto::ERROR;
727+
}
728+
});
729+
730+
env.SendSync(new NFake::TEvExecute{ new TTxDeleteRow(101, 42) });
731+
732+
env.SendSync(new NFake::TEvCall{ [](auto* executor, const auto& ctx) {
733+
executor->CleanupData(235);
734+
ctx.Send(ctx.SelfID, new NFake::TEvReturn);
735+
} });
736+
737+
// should not be cleaned without successful log GC
738+
UNIT_ASSERT(!env.GrabEdgeEvent<NFake::TEvDataCleaned>(TDuration::Seconds(10)));
739+
740+
// make all subsequent log GC responses OK
741+
obs.Remove();
742+
743+
auto ev2 = env.GrabEdgeEvent<NFake::TEvDataCleaned>();
744+
UNIT_ASSERT_VALUES_EQUAL(ev2->Get()->DataCleanupGeneration, 235);
745+
746+
UNIT_ASSERT_VALUES_EQUAL(BlobStorageValueCountInAllGroups(env, value42), 0);
747+
}
704748
}
705749
} // namespace NKikimr::NTable

0 commit comments

Comments
 (0)