Skip to content

Make cooldown timeout on VDisk flexible #4551

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 16, 2024
Merged

Make cooldown timeout on VDisk flexible #4551

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ydb/core/blobstorage/dsproxy/dsproxy_put.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ class TBlobStorageGroupPutRequest : public TBlobStorageGroupRequestActor<TBlobSt
}

void HandleIncarnation(TMonotonic timestamp, ui32 orderNumber, ui64 incarnationGuid) {
timestamp += TDuration::Seconds(15); // TODO: cooldown timeout
timestamp += VDiskCooldownTimeoutOnProxy;

Y_ABORT_UNLESS(orderNumber < IncarnationRecords.size());
auto& record = IncarnationRecords[orderNumber];
Expand Down
6 changes: 2 additions & 4 deletions ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@

namespace NKikimr::NStorage {

constexpr TDuration PDISK_CONFIDENCE_DELAY = TDuration::Seconds(15);

void TNodeWarden::DestroyLocalVDisk(TVDiskRecord& vdisk) {
STLOG(PRI_INFO, BS_NODE, NW35, "DestroyLocalVDisk", (VDiskId, vdisk.GetVDiskId()), (VSlotId, vdisk.GetVSlotId()));
Y_ABORT_UNLESS(!vdisk.RuntimeData);
Expand Down Expand Up @@ -296,7 +294,7 @@ namespace NKikimr::NStorage {
StartLocalVDiskActor(record, TDuration::Zero());
} else if (record.RuntimeData->DonorMode < record.Config.HasDonorMode() || record.RuntimeData->ReadOnly != record.Config.GetReadOnly()) {
PoisonLocalVDisk(record);
StartLocalVDiskActor(record, PDISK_CONFIDENCE_DELAY);
StartLocalVDiskActor(record, VDiskCooldownTimeout);
}
}

Expand All @@ -323,7 +321,7 @@ namespace NKikimr::NStorage {
auto& record = it->second;
if (record.GetVDiskId() == vDiskId) {
PoisonLocalVDisk(record);
StartLocalVDiskActor(record, PDISK_CONFIDENCE_DELAY);
StartLocalVDiskActor(record, VDiskCooldownTimeout);
break;
}
}
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/blobstorage/ut_blobstorage/vdisk_restart.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
Y_UNIT_TEST_SUITE(VDiskRestart) {

Y_UNIT_TEST(Simple) {
return;

TEnvironmentSetup env({
.NodeCount = 8,
.VDiskReplPausedAtStart = false,
Expand Down
34 changes: 4 additions & 30 deletions ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ namespace NKikimr {
bool HugeKeeperInitialized = false;
ui64 RecoveredLsn = 0;
ui64 SyncLogMaxLsnStored = 0;
bool CooldownTimerHit = false;
bool DatabaseLoaded = false;
NKikimrVDiskData::TScrubEntrypoint ScrubEntrypoint;
ui64 ScrubEntrypointLsn = 0;

Expand Down Expand Up @@ -215,11 +213,8 @@ namespace NKikimr {
ActiveActors.Erase(ev->Sender);
auto *msg = ev->Get();
RecoveredLsn = msg->RecoveredLsn;
if (RecoveredLsn) {
CooldownTimerHit = true; // we do not apply cooldown timer when there are entries in the log
}
if (msg->Status == NKikimrProto::OK) {
PostReplayRecoveryLog(ctx);
SignalSuccessAndDie(ctx);
} else {
SignalErrorAndDie(ctx, msg->Status, msg->ErrorReason);
}
Expand Down Expand Up @@ -560,19 +555,6 @@ namespace NKikimr {
AfterDatabaseLoaded(ctx);
}
}
void PostReplayRecoveryLog(const TActorContext &ctx) {
DatabaseLoaded = true;
if (DatabaseLoaded && CooldownTimerHit) {
SignalSuccessAndDie(ctx);
}
}

void HandleWakeup(const TActorContext &ctx) {
CooldownTimerHit = true;
if (DatabaseLoaded && CooldownTimerHit) {
SignalSuccessAndDie(ctx);
}
}

void SendYardInit(const TActorContext &ctx, TDuration yardInitDelay) {
auto ev = std::make_unique<NPDisk::TEvYardInit>(Config->BaseInfo.InitOwnerRound, SelfVDiskId,
Expand All @@ -596,8 +578,8 @@ namespace NKikimr {
LOG_NOTICE(ctx, BS_LOCALRECOVERY,
VDISKP(LocRecCtx->VCtx->VDiskLogPrefix, "LocalRecovery START"));

SendYardInit(ctx, Config->BaseInfo.YardInitDelay);
Become(&TThis::StateInitialize, ctx, VDiskCooldownTimeout, new TEvents::TEvWakeup);
SendYardInit(ctx, TDuration::Zero());
Become(&TThis::StateInitialize);
VDiskMonGroup.VDiskLocalRecoveryState() = TDbMon::TDbLocalRecovery::YardInit;
}

Expand Down Expand Up @@ -649,28 +631,24 @@ namespace NKikimr {
HFunc(TEvents::TEvUndelivered, HandleUndelivered)
CFunc(NActors::TEvents::TSystem::PoisonPill, HandlePoison)
HFunc(NMon::TEvHttpInfo, Handle)
CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
)

STRICT_STFUNC(StateLoadDatabase,
HFunc(THullIndexLoaded, Handle)
CFunc(NActors::TEvents::TSystem::PoisonPill, HandlePoison)
HFunc(NMon::TEvHttpInfo, Handle)
CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
)

STRICT_STFUNC(StateLoadBulkFormedSegments,
HFunc(TEvBulkSstsLoaded, Handle)
CFunc(NActors::TEvents::TSystem::PoisonPill, HandlePoison)
HFunc(NMon::TEvHttpInfo, Handle)
CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
)

STRICT_STFUNC(StateApplyRecoveryLog,
HFunc(TEvRecoveryLogReplayDone, Handle)
CFunc(NActors::TEvents::TSystem::PoisonPill, HandlePoison)
HFunc(NMon::TEvHttpInfo, Handle)
CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
)

public:
Expand All @@ -693,11 +671,7 @@ namespace NKikimr {
, LocRecCtx(std::make_shared<TLocalRecoveryContext>(vctx))
, Arena(std::move(arena))
, VDiskMonGroup(vctx->VDiskCounters, "subsystem", "state")
{
if (!Config->EnableVDiskCooldownTimeout || LocRecCtx->VCtx->Top->GType.BlobSubgroupSize() == 1) {
CooldownTimerHit = true;
}
}
{}
};


Expand Down
12 changes: 11 additions & 1 deletion ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,10 +298,19 @@ namespace NKikimr {
}

Become(&TThis::RecoverLostDataStateFunc);
Phase = TPhaseVal::PhaseRecoverLostData;

if (SyncerCtx->Config->EnableVDiskCooldownTimeout) {
Schedule(SyncerCtx->Config->BaseInfo.YardInitDelay, new TEvents::TEvWakeup);
} else {
RecoverLostDataResumeAfterDelay(ctx);
}
}

void RecoverLostDataResumeAfterDelay(const TActorContext& ctx) {
const TVDiskEternalGuid guid = GuidRecovOutcome->Guid;
RecoverLostDataId = ctx.Register(CreateSyncerRecoverLostDataActor(SyncerCtx, GInfo, CommitterId, ctx.SelfID, guid));
ActiveActors.Insert(RecoverLostDataId, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
Phase = TPhaseVal::PhaseRecoverLostData;
}

void Handle(TEvSyncerLostDataRecovered::TPtr &ev, const TActorContext &ctx) {
Expand All @@ -322,6 +331,7 @@ namespace NKikimr {
HFunc(TEvents::TEvPoisonPill, HandlePoison)
HFunc(TEvSublogLine, Handle)
HFunc(TEvVGenerationChange, RecoverLostDataModeHandle)
CFunc(TEvents::TSystem::Wakeup, RecoverLostDataResumeAfterDelay);
)

////////////////////////////////////////////////////////////////////////
Expand Down