Skip to content

Commit 2d522e2

Browse files
nshestakovnshestakov
committed
Merge branch 'main' into metrics
2 parents b1c1990 + 4b411f4 commit 2d522e2

File tree

92 files changed

+1673
-501
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

92 files changed

+1673
-501
lines changed

‎ydb/apps/dstool/lib/common.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ def invoke_grpc(func, *params, explicit_host=None, host=None):
329329
if connection_params.verbose:
330330
p = ', '.join('<<< %s >>>' % text_format.MessageToString(param, as_one_line=True) for param in params)
331331
print('INFO: issuing %s(%s) @%s:%d protocol %s' % (func, p, host, connection_params.grpc_port,
332-
connection_params.mon_protocol), file=sys.stderr)
332+
connection_params.mon_protocol), file=sys.stderr)
333333

334334
def work(channel):
335335
try:
@@ -354,6 +354,7 @@ def work(channel):
354354
retval = work(channel)
355355
return retval
356356

357+
357358
def invoke_bsc_request(request):
358359
if connection_params.http:
359360
tablet_id = 72057594037932033

‎ydb/apps/dstool/lib/dstool_cmd_cluster_set.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def add_options(p):
2626
g.add_argument('--disk-management-mode', type=str, choices=choices, help='Disk management mode')
2727
g.add_argument('--enable-self-heal-local-policy', action='store_const', const=True, dest='self_heal_local_policy', help='Enable SelfHeal local policy for cluster')
2828
g.add_argument('--disable-self-heal-local-policy', action='store_const', const=False, dest='self_heal_local_policy', help='Disable SelfHeal local policy for cluster')
29-
29+
3030
common.add_basic_format_options(p)
3131

3232

‎ydb/apps/dstool/lib/dstool_cmd_cluster_workload_run.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ def do_kill_blob_depot():
243243
if not args.disable_wipes:
244244
wipes.append(('wipe vslot id: %s, vdisk id: %s' % (vslot_id, vdisk_id), (do_wipe, vslot)))
245245
if not args.disable_readonly:
246-
readolies.append(('readonly vslot id: %s, vdisk id: %s' % (vslot_id, vdisk_id), (do_readonly, vslot, True)))
246+
readonlies.append(('readonly vslot id: %s, vdisk id: %s' % (vslot_id, vdisk_id), (do_readonly, vslot, True)))
247247

248248
def pick(v):
249249
action_name, action = random.choice(v)

‎ydb/core/blobstorage/vdisk/common/vdisk_context.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ namespace NKikimr {
125125
OutOfSpaceState.UpdateLocalLog(ev.StatusFlags);
126126
}
127127
return true;
128+
case NKikimrProto::ERROR:
128129
case NKikimrProto::INVALID_OWNER:
129130
case NKikimrProto::INVALID_ROUND:
130131
// BlobStorage group reconfiguration, just return false and wait until

‎ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.cpp‎

Lines changed: 198 additions & 120 deletions
Large diffs are not rendered by default.

‎ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge.h‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -209,13 +209,10 @@ namespace NKikimr {
209209
NHuge::THeapStat Stat;
210210
};
211211

212-
struct TEvHugePreCompact : TEventLocal<TEvHugePreCompact, TEvBlobStorage::EvHugePreCompact> {
213-
ui64 LsnInfimum; // the resulting LSN of the operation MUST be not less that the provided value
214-
TEvHugePreCompact(ui64 lsnInfimum) : LsnInfimum(lsnInfimum) {}
215-
};
212+
struct TEvHugePreCompact : TEventLocal<TEvHugePreCompact, TEvBlobStorage::EvHugePreCompact> {};
216213

217214
struct TEvHugePreCompactResult : TEventLocal<TEvHugePreCompactResult, TEvBlobStorage::EvHugePreCompactResult> {
218-
ui64 WId; // this is going to be provided in free slots operation
215+
const ui64 WId; // this is going to be provided in free slots operation
219216
TEvHugePreCompactResult(ui64 wId) : WId(wId) {}
220217
};
221218

‎ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugedefs.h‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,7 @@ namespace NKikimr {
129129
}
130130

131131
ui64 Hash() const {
132-
ui64 x = 0;
133-
x |= (ui64)ChunkId;
134-
x <<= 32u;
135-
x |= (ui64)Offset;
136-
return x;
132+
return MultiHash(ChunkId, Offset);
137133
}
138134

139135
void Serialize(IOutputStream &str) const {

‎ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ namespace NKikimr {
102102
maxBlobInBytes, overhead, freeChunksReservation))
103103
, AllocatedSlots()
104104
, Guid(TAppData::RandomProvider->GenRand64())
105+
, PersistentLsn(entryPointLsn)
105106
{
106107
ParseFromString(entryPointData);
107108
Y_ABORT_UNLESS(entryPointLsn == LogPos.EntryPointLsn);
@@ -129,6 +130,7 @@ namespace NKikimr {
129130
maxBlobInBytes, overhead, freeChunksReservation))
130131
, AllocatedSlots()
131132
, Guid(TAppData::RandomProvider->GenRand64())
133+
, PersistentLsn(entryPointLsn)
132134
{
133135
ParseFromArray(entryPointData.GetData(), entryPointData.GetSize());
134136
Y_ABORT_UNLESS(entryPointLsn == LogPos.EntryPointLsn);
@@ -304,7 +306,7 @@ namespace NKikimr {
304306
}
305307

306308
ui64 THullHugeKeeperPersState::FirstLsnToKeep(ui64 minInFlightLsn) const {
307-
const ui64 res = Min(minInFlightLsn, LogPos.EntryPointLsn);
309+
const ui64 res = Min(minInFlightLsn, PersistentLsn);
308310

309311
Y_VERIFY_S(FirstLsnToKeepReported <= res, "FirstLsnToKeepReported# " << FirstLsnToKeepReported
310312
<< " res# " << res << " state# " << FirstLsnToKeepDecomposed() << " minInFlightLsn# " << minInFlightLsn);
@@ -321,13 +323,18 @@ namespace NKikimr {
321323

322324
bool THullHugeKeeperPersState::WouldNewEntryPointAdvanceLog(ui64 freeUpToLsn, ui64 minInFlightLsn,
323325
ui32 itemsAfterCommit) const {
324-
return freeUpToLsn < minInFlightLsn && (LogPos.EntryPointLsn <= freeUpToLsn || itemsAfterCommit > 10000);
326+
return freeUpToLsn < minInFlightLsn && (PersistentLsn <= freeUpToLsn || itemsAfterCommit > 10000);
325327
}
326328

327329
// initiate commit
328-
void THullHugeKeeperPersState::InitiateNewEntryPointCommit(ui64 lsn) {
330+
void THullHugeKeeperPersState::InitiateNewEntryPointCommit(ui64 lsn, ui64 minInFlightLsn) {
329331
Y_ABORT_UNLESS(lsn > LogPos.EntryPointLsn);
330332
LogPos.EntryPointLsn = lsn;
333+
PersistentLsn = Min(lsn, minInFlightLsn);
334+
335+
// these metabases never have huge blobs and we never care about them actually
336+
LogPos.BlocksDbSlotDelLsn = lsn;
337+
LogPos.BarriersDbSlotDelLsn = lsn;
331338
}
332339

333340
// finish commit
@@ -349,6 +356,7 @@ namespace NKikimr {
349356
Guid, lsn, LogPos.EntryPointLsn, rec.ToString().data()));
350357
Heap->RecoveryModeAddChunk(rec.ChunkId);
351358
LogPos.ChunkAllocationLsn = lsn;
359+
PersistentLsn = Min(PersistentLsn, lsn);
352360
return TRlas(true, false);
353361
} else {
354362
// skip
@@ -376,6 +384,7 @@ namespace NKikimr {
376384
Guid, lsn, LogPos.EntryPointLsn, rec.ToString().data()));
377385
Heap->RecoveryModeRemoveChunks(rec.ChunkIds);
378386
LogPos.ChunkFreeingLsn = lsn;
387+
PersistentLsn = Min(PersistentLsn, lsn);
379388
return TRlas(true, false);
380389
} else {
381390
// skip
@@ -420,6 +429,7 @@ namespace NKikimr {
420429
Heap->RecoveryModeFree(x);
421430

422431
*logPosDelLsn = lsn;
432+
PersistentLsn = Min(PersistentLsn, lsn);
423433
return TRlas(true, false);
424434
} else {
425435
// skip
@@ -468,6 +478,7 @@ namespace NKikimr {
468478
Heap->RecoveryModeAllocate(rec.DiskAddr);
469479
}
470480
LogPos.HugeBlobLoggedLsn = lsn;
481+
PersistentLsn = Min(PersistentLsn, lsn);
471482
return TRlas(true, false);
472483
} else {
473484
// skip
@@ -485,7 +496,6 @@ namespace NKikimr {
485496
ui64 lsn,
486497
const TString &data)
487498
{
488-
489499
if (!CheckEntryPoint(data))
490500
return TRlas(false, true);
491501

@@ -508,7 +518,6 @@ namespace NKikimr {
508518
ui64 lsn,
509519
const TContiguousSpan &data)
510520
{
511-
512521
if (!CheckEntryPoint(data))
513522
return TRlas(false, true);
514523

@@ -526,7 +535,6 @@ namespace NKikimr {
526535
return TRlas(true, false);
527536
}
528537

529-
530538
void THullHugeKeeperPersState::FinishRecovery(const TActorContext &ctx) {
531539
// handle AllocatedSlots
532540
if (!AllocatedSlots.empty()) {

‎ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ namespace NKikimr {
8686
const ui64 Guid;
8787
// last reported FirstLsnToKeep; can't decrease
8888
mutable ui64 FirstLsnToKeepReported = 0;
89+
ui64 PersistentLsn = 0;
8990

9091
THullHugeKeeperPersState(TIntrusivePtr<TVDiskContext> vctx,
9192
const ui32 chunkSize,
@@ -138,7 +139,7 @@ namespace NKikimr {
138139
bool WouldNewEntryPointAdvanceLog(ui64 freeUpToLsn, ui64 minInFlightLsn, ui32 itemsAfterCommit) const;
139140

140141
// initiate commit
141-
void InitiateNewEntryPointCommit(ui64 lsn);
142+
void InitiateNewEntryPointCommit(ui64 lsn, ui64 minInFlightLsn);
142143
// finish commit
143144
void EntryPointCommitted(ui64 lsn);
144145

‎ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -288,9 +288,13 @@ namespace NKikimr {
288288
ApplyCompactionResult(ctx, {}, {}, 0);
289289
} else {
290290
const ui64 cookie = NextPreCompactCookie++;
291-
ctx.Send(HullLogCtx->HugeKeeperId, new TEvHugePreCompact(RTCtx->LsnMngr->GetLsn()), 0, cookie);
291+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, HullDs->HullCtx->VCtx->VDiskLogPrefix
292+
<< "requesting PreCompact for ActDeleteSsts");
293+
ctx.Send(HullLogCtx->HugeKeeperId, new TEvHugePreCompact, 0, cookie);
292294
PreCompactCallbacks.emplace(cookie, [this, ev](ui64 wId, const TActorContext& ctx) mutable {
293295
Y_ABORT_UNLESS(wId);
296+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, HullDs->HullCtx->VCtx->VDiskLogPrefix
297+
<< "got PreCompactResult for ActDeleteSsts, wId# " << wId);
294298
ApplyCompactionResult(ctx, {}, {}, wId);
295299
RTCtx->LevelIndex->UpdateLevelStat(LevelStat);
296300
});
@@ -439,9 +443,13 @@ namespace NKikimr {
439443

440444
if (!msg->FreedHugeBlobs.Empty() && !wId && !msg->Aborted) {
441445
const ui64 cookie = NextPreCompactCookie++;
442-
ctx.Send(HullLogCtx->HugeKeeperId, new TEvHugePreCompact(RTCtx->LsnMngr->GetLsn()), 0, cookie);
446+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, HullDs->HullCtx->VCtx->VDiskLogPrefix
447+
<< "requesting PreCompact for THullChange");
448+
ctx.Send(HullLogCtx->HugeKeeperId, new TEvHugePreCompact, 0, cookie);
443449
PreCompactCallbacks.emplace(cookie, [this, ev](ui64 wId, const TActorContext& ctx) mutable {
444450
Y_ABORT_UNLESS(wId);
451+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, HullDs->HullCtx->VCtx->VDiskLogPrefix
452+
<< "got PreCompactResult for THullChange, wId# " << wId);
445453
Handle(ev, ctx, wId);
446454
});
447455
return;
@@ -455,7 +463,7 @@ namespace NKikimr {
455463
if (msg->FreshSegment) {
456464
TStringStream dbg;
457465
dbg << "{commiter# fresh"
458-
<< " firtsLsn# "<< msg->FreshSegment->GetFirstLsn()
466+
<< " firstLsn# "<< msg->FreshSegment->GetFirstLsn()
459467
<< " lastLsn# " << msg->FreshSegment->GetLastLsn()
460468
<< "}";
461469

@@ -514,6 +522,7 @@ namespace NKikimr {
514522
const auto it = PreCompactCallbacks.find(ev->Cookie);
515523
Y_ABORT_UNLESS(it != PreCompactCallbacks.end());
516524
it->second(ev->Get()->WId, ctx);
525+
PreCompactCallbacks.erase(it);
517526
}
518527

519528
void Handle(typename TFreshAppendixCompactionDone::TPtr& ev, const TActorContext& ctx) {

0 commit comments

Comments
 (0)