Tmp debug compaction

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_balance.h

@@ -73,6 +73,9 @@ namespace NKikimr {
 
                         // put all found ssts into Vec
                         CompactSsts.PushSstFromLevelX(level, trgtFirstIt, trgtEndIt);
+                        LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                            HullCtx->VCtx->VDiskLogPrefix << " TBalanceBase::FindNeighborhoods decided to compact to level# " << level
+                            << " Task# " << CompactSsts.ToString());
                     } else {
                         // we don't have any ssts at level, it's fine
                     }
@@ -118,8 +121,7 @@ namespace NKikimr {
             // find empty level to put compaction result to
             ui32 FindTargetLevel() const {
                 ui32 otherLevelsNum = SliceSnap.GetLevelXNumber();
-                ui32 i = 0;
-                for (; i < Boundaries->SortedParts * 2; i++) {
+                for (ui32 i = 0; i < Boundaries->SortedParts * 2; i++) {
                     if (i < otherLevelsNum) {
                         const TSortedLevel &sl = SliceSnap.GetLevelXRef(i);
                         if (sl.Empty()) {
@@ -149,6 +151,9 @@ namespace NKikimr {
                     it.Next();
                 }
                 Y_ABORT_UNLESS(added > 0);
+
+                LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                        HullCtx->VCtx->VDiskLogPrefix << " TBalanceLevel0 decided to compact, Task# " << CompactSsts.ToString());
             }
 
             // check and run full compaction if required
@@ -234,9 +239,10 @@ namespace NKikimr {
                 if (HullCtx->VCtx->ActorSystem) {
                     LOG_DEBUG(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
                             VDISKP(HullCtx->VCtx->VDiskLogPrefix,
-                                "%s: TBalancePartiallySortedLevels::CalculateRank: %s",
+                                "%s: TBalancePartiallySortedLevels::CalculateRank: %s, "
+                                "freeLevels: %" PRIu32 ", totalPsl %" PRIu32,
                                 PDiskSignatureForHullDbKey<TKey>().ToString().data(),
-                                ToString().data()));
+                                ToString().data(), freeLevels, totalPsl));
                 }
 
                 Y_ABORT_UNLESS(freeLevels <= totalPsl);
@@ -310,6 +316,12 @@ namespace NKikimr {
                     added++;
                 }
                 Y_ABORT_UNLESS(added);
+
+                LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                    HullCtx->VCtx->VDiskLogPrefix << " TBalancePartiallySortedLevels decided to compact, Task# " << CompactSsts.ToString()
+                    << " firstKeyToCover# " << (firstKeyToCover ? firstKeyToCover->ToString() : "nullptr")
+                    << " lastKeyToCover# " << (lastKeyToCover ? lastKeyToCover->ToString() : "nullptr")
+                );
             }
 
             void Compact() {
@@ -419,6 +431,11 @@ namespace NKikimr {
 
                 Y_ABORT_UNLESS(srcIt != srcSegs.end());
                 CompactSst(srcLevel, srcIt);
+
+                LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                    HullCtx->VCtx->VDiskLogPrefix << " TBalanceLevelX decided to compact, Task# " << CompactSsts.ToString()
+                    << " lastCompactedKey# " << lastCompactedKey.ToString()
+                );
             }
 
             void CompactSst(ui32 srcLevel, typename TSegments::const_iterator srcIt) {
@@ -427,6 +444,12 @@ namespace NKikimr {
                 TKey firstKeyToCover = (*srcIt)->FirstKey();
                 TKey lastKeyToCover = (*srcIt)->LastKey();
 
+
+                LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                    HullCtx->VCtx->VDiskLogPrefix << " TBalanceLevelX: take sst with firstKeyToCover#" << firstKeyToCover.ToString()
+                    << " lastKeyToCover# " << lastKeyToCover.ToString()
+                );
+
                 // put this sst to the vector
                 CompactSsts.TargetLevel = srcLevel + 1;
                 CompactSsts.PushSstFromLevelX(srcLevel, srcIt, srcIt + 1);
@@ -582,8 +605,8 @@ namespace NKikimr {
 
                 TString ToString() const {
                     TStringStream str;
-                    str  << "{VirtLevelComp# " << VirtualLevelToCompact
-                    << " RreePartSortLevels# " << FreePartiallySortedLevelsNum
+                    str  << "{VirtualLevelToCompact# " << VirtualLevelToCompact
+                    << " FreePartiallySortedLevelsNum# " << FreePartiallySortedLevelsNum
                     << " Ranks# ";
                     for (const auto &x : *this)
                         str << " " << x;
@@ -603,16 +626,16 @@ namespace NKikimr {
                 // find level to compact
                 Y_DEBUG_ABORT_UNLESS(!ranks.empty());
                 ranks.VirtualLevelToCompact = 0;
-                double minRank = ranks[0];
+                double maxRank = ranks[0];
                 for (ui32 i = 1; i < ranks.size(); i++) {
-                    if (ranks[i] > minRank) {
-                        minRank = ranks[i];
+                    if (ranks[i] > maxRank) {
+                        maxRank = ranks[i];
                         ranks.VirtualLevelToCompact = i;
                     }
                 }
 
                 // fill in compaction task or do nothing
-                if (minRank < RankThreshold) {
+                if (maxRank < RankThreshold) {
                     if (FullCompactionAttrs) {
                         if (BalanceLevel0.FullCompact(FullCompactionAttrs->FullCompactionLsn))
                             return ActCompactSsts;
@@ -632,6 +655,8 @@ namespace NKikimr {
                         return ActNothing;
                     }
                 } else {
+                    LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                            HullCtx->VCtx->VDiskLogPrefix << " TStrategyBalance decided to compact, ranks# " << ranks.ToString());
                     switch (ranks.VirtualLevelToCompact) {
                         case 0:     BalanceLevel0.Compact(); break;
                         case 1:     BalancePartiallySortedLevels.Compact(); break;

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_defs.h

@@ -268,6 +268,12 @@ namespace NKikimr {
                     TBase::Output(str);
                 }
 
+                TString ToString() const {
+                    TStringStream str;
+                    Output(str);
+                    return str.Str();
+                }
+
                 void Finalize() {
                     TBase::Finalize();
                     Y_ABORT_UNLESS(TargetLevel != (ui32)(-1));

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_delsst.h

@@ -66,6 +66,8 @@ namespace NKikimr {
                     TSstRatioPtr ratio = p.SstPtr->StorageRatio.Get();
                     if (p.Level > 0 && ratio && ratio->CanDeleteSst()) {
                         action = ActDeleteSsts;
+                        LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                            HullCtx->VCtx->VDiskLogPrefix << " TStrategyDelSst going to delete SST# " << p.ToString() << " because of ration# " << ratio->ToString());
                         Task->DeleteSsts.DeleteSst(p.Level, p.SstPtr);
                         SstToDelete++;
                     }

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_promote.h

@@ -67,7 +67,7 @@ namespace NKikimr {
                 it.SeekToFirst();
                 while (it.Valid()) {
                     TLevelSstPtr p = it.Get();
-                    ui32 level = p.Level;
+                    const ui32 level = p.Level;
 
                     if (level >= 2 * Boundaries->SortedParts + 1) {
                         TLevelSegmentPtr sst = p.SstPtr;
@@ -78,6 +78,8 @@ namespace NKikimr {
                         if (!DoIntersect(sst, nextLevel.Segs)) {
                             action = ActMoveSsts;
                             Task->MoveSsts.MoveSst(level, level + 1, sst);
+                            LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                                HullCtx->VCtx->VDiskLogPrefix << " TStrategyPromoteSsts: move sst# " << p.ToString() << " to level " << level + 1);
                             break;
                         }
                     }

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.cpp

@@ -124,4 +124,3 @@ namespace NKikimr {
 
     } // NHullComp
 } // NKikimr
-

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_selector.h

@@ -109,7 +109,8 @@ namespace NKikimr {
                 ctx.Send(RecipientID, new TSelected(action, std::move(CompactionTask)));
 
                 TInstant finishTime(TAppData::TimeProvider->Now());
-                LOG_INFO(ctx, NKikimrServices::BS_HULLCOMP,
+                LOG_LOG(ctx, action == ActNothing ? NLog::PRI_DEBUG : NLog::PRI_INFO,
+                    NKikimrServices::BS_HULLCOMP,
                          VDISKP(HullCtx->VCtx->VDiskLogPrefix,
                                 "%s: Selector actor: action# %s timeSpent# %s",
                                 PDiskSignatureForHullDbKey<TKey>().ToString().data(),

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_space.h

@@ -123,6 +123,9 @@ namespace NKikimr {
 
                 if (Candidate.CompactSstToFreeSpace()) {
                     // free space by compacting this Sst
+                    LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                            HullCtx->VCtx->VDiskLogPrefix << " TStrategyFreeSpace decided to compact Ssts " << Task->CompactSsts.ToString()
+                            << " because of high garbage/data retio " << Candidate.ToString());
                     action = ActCompactSsts;
                     TUtils::SqueezeOneSst(LevelSnap.SliceSnap, Candidate.LevelSstPtr, Task->CompactSsts);
                 }
@@ -133,4 +136,3 @@ namespace NKikimr {
 
     } // NHullComp
 } // NKikimr
-

ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_squeeze.h

@@ -75,6 +75,8 @@ namespace NKikimr {
                     TLevelSstPtr p = it.Get();
                     if (p.Level > 0) {
                         if (p.SstPtr->Info.CTime < SqueezeBefore) {
+                            LOG_INFO_S(*HullCtx->VCtx->ActorSystem, NKikimrServices::BS_HULLCOMP,
+                                HullCtx->VCtx->VDiskLogPrefix << " TStrategySqueeze decided to compact Sst " << p.ToString());
                             // rewrite this SST squeezed
                             TUtils::SqueezeOneSst(LevelSnap.SliceSnap, p, Task->CompactSsts);
                             return ActCompactSsts;
@@ -90,4 +92,3 @@ namespace NKikimr {
 
     } // NHullComp
 } // NKikimr
-

ydb/core/blobstorage/vdisk/hulldb/generic/hullds_sst.h

@@ -40,7 +40,7 @@ namespace NKikimr {
             }
 
             TString ToString() const {
-                return Sprintf("[%u %p %s]", Level, SstPtr.Get(), SstPtr->FirstKey().ToString().data());
+                return Sprintf("[%u %p (%s-%s)]", Level, SstPtr.Get(), SstPtr->FirstKey().ToString().data(), SstPtr->LastKey().ToString().data());
             }
 
         private:

ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp

@@ -206,6 +206,7 @@ namespace NKikimr {
         ReplayAddLogoBlobCmd(ctx, id, partId, ingress, std::move(buffer), lsn, THullDbRecovery::NORMAL);
 
         // run compaction if required
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddLogoBlob"));
         CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
     }
@@ -220,6 +221,7 @@ namespace NKikimr {
         ReplayAddHugeLogoBlobCmd(ctx, id, ingress, diskAddr, lsn, THullDbRecovery::NORMAL);
 
         // run compaction if required
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddHugeLogoBlob"));
         CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
     }
@@ -234,6 +236,7 @@ namespace NKikimr {
         ReplayAddLogoBlobCmd(ctx, id, ingress, seg.Point(), THullDbRecovery::NORMAL);
 
         // run compaction if required
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddLogoBlob with seg"));
         CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
     }
@@ -280,6 +283,7 @@ namespace NKikimr {
         Fields->DelayedResponses.ConfirmLsn(lsn, replySender);
 
         // run compaction if required
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Blocks in AddBlockCmd"));
         CompactFreshSegmentIfRequired<TKeyBlock, TMemRecBlock>(HullDs, Fields->BlocksRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
     }
@@ -474,8 +478,10 @@ namespace NKikimr {
         ReplayAddGCCmd(ctx, record, ingress, seg.Last);
 
         // run compaction if required
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddGCCmd"));
         CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Barriers in AddGCCmd"));
         CompactFreshSegmentIfRequired<TKeyBarrier, TMemRecBarrier>(HullDs, Fields->BarriersRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
     }
@@ -615,10 +621,13 @@ namespace NKikimr {
         Y_DEBUG_ABORT_UNLESS(curLsn == seg.Last + 1);
 
         // run compaction if required
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of LogoBlobs in AddSyncDataCmd"));
         CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->LogoBlobsRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Blocks in AddSyncDataCmd"));
         CompactFreshSegmentIfRequired<TKeyBlock, TMemRecBlock>(HullDs, Fields->BlocksRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
+        LOG_DEBUG(ctx, NKikimrServices::BS_HULLRECS, VDISKP(HullDs->HullCtx->VCtx->VDiskLogPrefix, "Try to schedule fresh compaction of Barriers in AddSyncDataCmd"));
         CompactFreshSegmentIfRequired<TKeyBarrier, TMemRecBarrier>(HullDs, Fields->BarriersRunTimeCtx, ctx, false,
             Fields->AllowGarbageCollection);
     }

ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp

@@ -225,6 +225,7 @@ namespace NKikimr {
 
         void ScheduleCompaction(const TActorContext &ctx) {
             // schedule fresh if required
+            LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "Try to schedule fresh compaction because of scheduling general compaction");
             CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, RTCtx, ctx, FullCompactionState.ForceFreshCompaction(RTCtx),
                 AllowGarbageCollection);
             if (!Config->BaseInfo.ReadOnly && !RunLevelCompactionSelector(ctx)) {
@@ -562,6 +563,7 @@ namespace NKikimr {
                     if (FullCompactionState.Enabled()) {
                         ScheduleCompaction(ctx);
                     } else {
+                        LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "Try to schedule fresh compaction in Handle(THullCommitFinished)");
                         CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, RTCtx, ctx,
                             FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
                     }
@@ -586,6 +588,7 @@ namespace NKikimr {
             const ui64 freeUpToLsn = ev->Get()->FreeUpToLsn;
             RTCtx->SetFreeUpToLsn(freeUpToLsn);
             // we check if we need to start fresh compaction, FreeUpToLsn influence our decision
+            LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "Try to schedule fresh compaction in Handle(NPDisk::TEvCutLog)");
             const bool freshCompStarted = CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, RTCtx, ctx,
                 FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
             // just for valid info output to the log

ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h

@@ -92,7 +92,18 @@ namespace NKikimr {
             bool allowGarbageCollection)
     {
         ui64 yardFreeUpToLsn = rtCtx->GetFreeUpToLsn();
-        bool compact = hullDs->HullCtx->FreshCompaction && rtCtx->LevelIndex->NeedsFreshCompaction(yardFreeUpToLsn, force);
+        bool needsFreshCompaction = false;
+        bool compact = false;
+        if (hullDs->HullCtx->FreshCompaction) {
+            needsFreshCompaction = rtCtx->LevelIndex->NeedsFreshCompaction(yardFreeUpToLsn, force);
+            compact = needsFreshCompaction;
+        }
+        LOG_DEBUG_S(ctx, NKikimrServices::BS_HULLCOMP, "CompactFreshSegmentIfRequired"
+            << ", required: " << compact
+            << ", yardFreeUpToLsn: " << yardFreeUpToLsn
+            << ", force: " << force
+            << ", needsFreshCompaction: " << needsFreshCompaction
+            << ", allowGarbageCollection: " << allowGarbageCollection);
         if (compact) {
             CompactFreshSegment<TKey, TMemRec>(hullDs, rtCtx, ctx, allowGarbageCollection);
         }