Skip to content

Commit a32fb9b

Browse files
VPolkaVPolka
authored
Merge 3fd196a into 188cfc1 
2 parents 188cfc1 + 3fd196a commit a32fb9b

12 files changed

+78
-17
lines changed

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_balance.h

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@ namespace NKikimr {
7373

7474
// put all found ssts into Vec
7575
CompactSsts.PushSstFromLevelX(level, trgtFirstIt, trgtEndIt);
76+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
77+
HullCtx->VCtx->VDiskLogPrefix << " TBalanceBase::FindNeighborhoods decided to compact to level# " << level
78+
<< " Task# " << CompactSsts.ToString());
7679
} else {
7780
// we don't have any ssts at level, it's fine
7881
}
@@ -118,8 +121,7 @@ namespace NKikimr {
118121
// find empty level to put compaction result to
119122
ui32 FindTargetLevel() const {
120123
ui32 otherLevelsNum = SliceSnap.GetLevelXNumber();
121-
ui32 i = 0;
122-
for (; i < Boundaries->SortedParts * 2; i++) {
124+
for (ui32 i = 0; i < Boundaries->SortedParts * 2; i++) {
123125
if (i < otherLevelsNum) {
124126
const TSortedLevel &sl = SliceSnap.GetLevelXRef(i);
125127
if (sl.Empty()) {
@@ -149,6 +151,9 @@ namespace NKikimr {
149151
it.Next();
150152
}
151153
Y_ABORT_UNLESS(added > 0);
154+
155+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
156+
HullCtx->VCtx->VDiskLogPrefix << " TBalanceLevel0 decided to compact, Task# " << CompactSsts.ToString());
152157
}
153158

154159
// check and run full compaction if required
@@ -234,9 +239,10 @@ namespace NKikimr {
234239
if (HullCtx->VCtx->ActorSystem) {
235240
LOG_DEBUG(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
236241
VDISKP(HullCtx->VCtx->VDiskLogPrefix,
237-
"%s: TBalancePartiallySortedLevels::CalculateRank: %s",
242+
"%s: TBalancePartiallySortedLevels::CalculateRank: %s, "
243+
"freeLevels: %" PRIu32 ", totalPsl %" PRIu32,
238244
PDiskSignatureForHullDbKey<TKey>().ToString().data(),
239-
ToString().data()));
245+
ToString().data(), freeLevels, totalPsl));
240246
}
241247

242248
Y_ABORT_UNLESS(freeLevels <= totalPsl);
@@ -310,6 +316,12 @@ namespace NKikimr {
310316
added++;
311317
}
312318
Y_ABORT_UNLESS(added);
319+
320+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
321+
HullCtx->VCtx->VDiskLogPrefix << " TBalancePartiallySortedLevels decided to compact, Task# " << CompactSsts.ToString()
322+
<< " firstKeyToCover# " << (firstKeyToCover ? firstKeyToCover->ToString() : "nullptr")
323+
<< " lastKeyToCover# " << (lastKeyToCover ? lastKeyToCover->ToString() : "nullptr")
324+
);
313325
}
314326

315327
void Compact() {
@@ -419,6 +431,11 @@ namespace NKikimr {
419431

420432
Y_ABORT_UNLESS(srcIt != srcSegs.end());
421433
CompactSst(srcLevel, srcIt);
434+
435+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
436+
HullCtx->VCtx->VDiskLogPrefix << " TBalanceLevelX decided to compact, Task# " << CompactSsts.ToString()
437+
<< " lastCompactedKey# " << lastCompactedKey.ToString()
438+
);
422439
}
423440

424441
void CompactSst(ui32 srcLevel, typename TSegments::const_iterator srcIt) {
@@ -427,6 +444,12 @@ namespace NKikimr {
427444
TKey firstKeyToCover = (*srcIt)->FirstKey();
428445
TKey lastKeyToCover = (*srcIt)->LastKey();
429446

447+
448+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
449+
HullCtx->VCtx->VDiskLogPrefix << " TBalanceLevelX: take sst with firstKeyToCover#" << firstKeyToCover.ToString()
450+
<< " lastKeyToCover# " << lastKeyToCover.ToString()
451+
);
452+
430453
// put this sst to the vector
431454
CompactSsts.TargetLevel = srcLevel + 1;
432455
CompactSsts.PushSstFromLevelX(srcLevel, srcIt, srcIt + 1);
@@ -582,8 +605,8 @@ namespace NKikimr {
582605

583606
TString ToString() const {
584607
TStringStream str;
585-
str << "{VirtLevelComp# " << VirtualLevelToCompact
586-
<< " RreePartSortLevels# " << FreePartiallySortedLevelsNum
608+
str << "{VirtualLevelToCompact# " << VirtualLevelToCompact
609+
<< " FreePartiallySortedLevelsNum# " << FreePartiallySortedLevelsNum
587610
<< " Ranks# ";
588611
for (const auto &x : *this)
589612
str << " " << x;
@@ -603,16 +626,16 @@ namespace NKikimr {
603626
// find level to compact
604627
Y_DEBUG_ABORT_UNLESS(!ranks.empty());
605628
ranks.VirtualLevelToCompact = 0;
606-
double minRank = ranks[0];
629+
double maxRank = ranks[0];
607630
for (ui32 i = 1; i < ranks.size(); i++) {
608-
if (ranks[i] > minRank) {
609-
minRank = ranks[i];
631+
if (ranks[i] > maxRank) {
632+
maxRank = ranks[i];
610633
ranks.VirtualLevelToCompact = i;
611634
}
612635
}
613636

614637
// fill in compaction task or do nothing
615-
if (minRank < RankThreshold) {
638+
if (maxRank < RankThreshold) {
616639
if (FullCompactionAttrs) {
617640
if (BalanceLevel0.FullCompact(FullCompactionAttrs->FullCompactionLsn))
618641
return ActCompactSsts;
@@ -632,6 +655,8 @@ namespace NKikimr {
632655
return ActNothing;
633656
}
634657
} else {
658+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
659+
HullCtx->VCtx->VDiskLogPrefix << " TStrategyBalance decided to compact, ranks# " << ranks.ToString());
635660
switch (ranks.VirtualLevelToCompact) {
636661
case 0: BalanceLevel0.Compact(); break;
637662
case 1: BalancePartiallySortedLevels.Compact(); break;

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_defs.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,12 @@ namespace NKikimr {
268268
TBase::Output(str);
269269
}
270270

271+
TString ToString() const {
272+
TStringStream str;
273+
Output(str);
274+
return str.Str();
275+
}
276+
271277
void Finalize() {
272278
TBase::Finalize();
273279
Y_ABORT_UNLESS(TargetLevel != (ui32)(-1));

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_delsst.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ namespace NKikimr {
6666
TSstRatioPtr ratio = p.SstPtr->StorageRatio.Get();
6767
if (p.Level > 0 && ratio && ratio->CanDeleteSst()) {
6868
action = ActDeleteSsts;
69+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
70+
HullCtx->VCtx->VDiskLogPrefix << " TStrategyDelSst going to delete SST# " << p.ToString() << " because of ration# " << ratio->ToString());
6971
Task->DeleteSsts.DeleteSst(p.Level, p.SstPtr);
7072
SstToDelete++;
7173
}

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_promote.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ namespace NKikimr {
6767
it.SeekToFirst();
6868
while (it.Valid()) {
6969
TLevelSstPtr p = it.Get();
70-
ui32 level = p.Level;
70+
const ui32 level = p.Level;
7171

7272
if (level >= 2 * Boundaries->SortedParts + 1) {
7373
TLevelSegmentPtr sst = p.SstPtr;
@@ -78,6 +78,8 @@ namespace NKikimr {
7878
if (!DoIntersect(sst, nextLevel.Segs)) {
7979
action = ActMoveSsts;
8080
Task->MoveSsts.MoveSst(level, level + 1, sst);
81+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
82+
HullCtx->VCtx->VDiskLogPrefix << " TStrategyPromoteSsts: move sst# " << p.ToString() << " to level " << level + 1);
8183
break;
8284
}
8385
}

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,4 +124,3 @@ namespace NKikimr {
124124

125125
} // NHullComp
126126
} // NKikimr
127-

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,8 @@ namespace NKikimr {
109109
ctx.Send(RecipientID, new TSelected(action, std::move(CompactionTask)));
110110

111111
TInstant finishTime(TAppData::TimeProvider->Now());
112-
LOG_INFO(ctx, NKikimrServices::BS_HULLCOMP,
112+
LOG_LOG(ctx, action == ActNothing ? NLog::PRI_DEBUG : NLog::PRI_INFO,
113+
NKikimrServices::BS_HULLCOMP,
113114
VDISKP(HullCtx->VCtx->VDiskLogPrefix,
114115
"%s: Selector actor: action# %s timeSpent# %s",
115116
PDiskSignatureForHullDbKey<TKey>().ToString().data(),

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_space.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ namespace NKikimr {
123123

124124
if (Candidate.CompactSstToFreeSpace()) {
125125
// free space by compacting this Sst
126+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
127+
HullCtx->VCtx->VDiskLogPrefix << " TStrategyFreeSpace decided to compact Ssts " << Task->CompactSsts.ToString()
128+
<< " because of high garbage/data retio " << Candidate.ToString());
126129
action = ActCompactSsts;
127130
TUtils::SqueezeOneSst(LevelSnap.SliceSnap, Candidate.LevelSstPtr, Task->CompactSsts);
128131
}
@@ -133,4 +136,3 @@ namespace NKikimr {
133136

134137
} // NHullComp
135138
} // NKikimr
136-

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_squeeze.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ namespace NKikimr {
7575
TLevelSstPtr p = it.Get();
7676
if (p.Level > 0) {
7777
if (p.SstPtr->Info.CTime < SqueezeBefore) {
78+
LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
79+
HullCtx->VCtx->VDiskLogPrefix << " TStrategySqueeze decided to compact Sst " << p.ToString());
7880
// rewrite this SST squeezed
7981
TUtils::SqueezeOneSst(LevelSnap.SliceSnap, p, Task->CompactSsts);
8082
return ActCompactSsts;
@@ -90,4 +92,3 @@ namespace NKikimr {
9092

9193
} // NHullComp
9294
} // NKikimr
93-

‎ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sst.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ namespace NKikimr {
4040
}
4141

4242
TString ToString() const {
43-
return Sprintf("[%u %p %s]", Level, SstPtr.Get(), SstPtr->FirstKey().ToString().data());
43+
return Sprintf("[%u %p (%s-%s)]", Level, SstPtr.Get(), SstPtr->FirstKey().ToString().data(), SstPtr->LastKey().ToString().data());
4444
}
4545

4646
private:

‎ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ namespace NKikimr {
206206
ReplayAddLogoBlobCmd(ctx, id, partId, ingress, std::move(buffer), lsn, THullDbRecovery::NORMAL);
207207

208208
// run compaction if required
209+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddLogoBlob"));
209210
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
210211
Fields->AllowGarbageCollection);
211212
}
@@ -220,6 +221,7 @@ namespace NKikimr {
220221
ReplayAddHugeLogoBlobCmd(ctx, id, ingress, diskAddr, lsn, THullDbRecovery::NORMAL);
221222

222223
// run compaction if required
224+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddHugeLogoBlob"));
223225
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
224226
Fields->AllowGarbageCollection);
225227
}
@@ -234,6 +236,7 @@ namespace NKikimr {
234236
ReplayAddLogoBlobCmd(ctx, id, ingress, seg.Point(), THullDbRecovery::NORMAL);
235237

236238
// run compaction if required
239+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddLogoBlob with seg"));
237240
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
238241
Fields->AllowGarbageCollection);
239242
}
@@ -280,6 +283,7 @@ namespace NKikimr {
280283
Fields->DelayedResponses.ConfirmLsn(lsn, replySender);
281284

282285
// run compaction if required
286+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Blocks in AddBlockCmd"));
283287
CompactFreshSegmentIfRequired<TKeyBlock, TMemRecBlock>(HullDs, Fields->BlocksRunTimeCtx, ctx, false,
284288
Fields->AllowGarbageCollection);
285289
}
@@ -474,8 +478,10 @@ namespace NKikimr {
474478
ReplayAddGCCmd(ctx, record, ingress, seg.Last);
475479

476480
// run compaction if required
481+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddGCCmd"));
477482
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
478483
Fields->AllowGarbageCollection);
484+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Barriers in AddGCCmd"));
479485
CompactFreshSegmentIfRequired<TKeyBarrier, TMemRecBarrier>(HullDs, Fields->BarriersRunTimeCtx, ctx, false,
480486
Fields->AllowGarbageCollection);
481487
}
@@ -615,10 +621,13 @@ namespace NKikimr {
615621
Y_DEBUG_ABORT_UNLESS(curLsn == seg.Last + 1);
616622

617623
// run compaction if required
624+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddSyncDataCmd"));
618625
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
619626
Fields->AllowGarbageCollection);
627+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Blocks in AddSyncDataCmd"));
620628
CompactFreshSegmentIfRequired<TKeyBlock, TMemRecBlock>(HullDs, Fields->BlocksRunTimeCtx, ctx, false,
621629
Fields->AllowGarbageCollection);
630+
LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Barriers in AddSyncDataCmd"));
622631
CompactFreshSegmentIfRequired<TKeyBarrier, TMemRecBarrier>(HullDs, Fields->BarriersRunTimeCtx, ctx, false,
623632
Fields->AllowGarbageCollection);
624633
}

‎ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ namespace NKikimr {
225225

226226
void ScheduleCompaction(const TActorContext &ctx) {
227227
// schedule fresh if required
228+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "Try to schedule fresh compaction because of scheduling general compaction");
228229
CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, RTCtx, ctx, FullCompactionState.ForceFreshCompaction(RTCtx),
229230
AllowGarbageCollection);
230231
if (!Config->BaseInfo.ReadOnly && !RunLevelCompactionSelector(ctx)) {
@@ -562,6 +563,7 @@ namespace NKikimr {
562563
if (FullCompactionState.Enabled()) {
563564
ScheduleCompaction(ctx);
564565
} else {
566+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "Try to schedule fresh compaction in Handle(THullCommitFinished)");
565567
CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, RTCtx, ctx,
566568
FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
567569
}
@@ -586,6 +588,7 @@ namespace NKikimr {
586588
const ui64 freeUpToLsn = ev->Get()->FreeUpToLsn;
587589
RTCtx->SetFreeUpToLsn(freeUpToLsn);
588590
// we check if we need to start fresh compaction, FreeUpToLsn influence our decision
591+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "Try to schedule fresh compaction in Handle(NPDisk::TEvCutLog)");
589592
const bool freshCompStarted = CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, RTCtx, ctx,
590593
FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
591594
// just for valid info output to the log

‎ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,18 @@ namespace NKikimr {
9292
bool allowGarbageCollection)
9393
{
9494
ui64 yardFreeUpToLsn = rtCtx->GetFreeUpToLsn();
95-
bool compact = hullDs->HullCtx->FreshCompaction && rtCtx->LevelIndex->NeedsFreshCompaction(yardFreeUpToLsn, force);
95+
bool needsFreshCompaction = false;
96+
bool compact = false;
97+
if (hullDs->HullCtx->FreshCompaction) {
98+
needsFreshCompaction = rtCtx->LevelIndex->NeedsFreshCompaction(yardFreeUpToLsn, force);
99+
compact = needsFreshCompaction;
100+
}
101+
LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "CompactFreshSegmentIfRequired"
102+
<< ", required: " << compact
103+
<< ", yardFreeUpToLsn: " << yardFreeUpToLsn
104+
<< ", force: " << force
105+
<< ", needsFreshCompaction: " << needsFreshCompaction
106+
<< ", allowGarbageCollection: " << allowGarbageCollection);
96107
if (compact) {
97108
CompactFreshSegment<TKey, TMemRec>(hullDs, rtCtx, ctx, allowGarbageCollection);
98109
}

0 commit comments

Comments
 (0)