Skip to content

Commit 14bc385

Browse files
alexvrualexvru
authored
Merge 69080e7 into f7b2a35 
2 parents f7b2a35 + 69080e7 commit 14bc385

15 files changed

+58
-46
lines changed

‎ydb/core/blobstorage/vdisk/anubis_osiris/blobstorage_anubisfinder.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,7 @@ namespace NKikimr {
8989
// calculate keep status
9090
bool allowKeepFlags = HullCtx->AllowKeepFlags;
9191
NGc::TKeepStatus keep = brs->Keep(dbIt.GetCurKey(), dbMerger.GetMemRec(),
92-
dbMerger.GetMemRecsMerged(), allowKeepFlags,
93-
true /*allowGarbageCollection*/);
92+
{subsMerger, dbMerger}, allowKeepFlags, true /*allowGarbageCollection*/);
9493
if (keep.KeepIndex && !keep.KeepByBarrier) {
9594
// we keep this record because of keep flags
9695
candidates.AddCandidate(dbIt.GetCurKey().LogoBlobID());

‎ydb/core/blobstorage/vdisk/anubis_osiris/blobstorage_osiris.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,8 @@ namespace NKikimr {
2222

2323
bool Check(const TKeyLogoBlob &key,
2424
const TMemRecLogoBlob &memRec,
25-
ui32 recsMerged,
2625
bool allowKeepFlags) const {
27-
return BarriersEssence->Keep(key, memRec, recsMerged, allowKeepFlags, false /*allowGarbageCollection*/).KeepData;
26+
return BarriersEssence->Keep(key, memRec, {}, allowKeepFlags, false /*allowGarbageCollection*/).KeepData;
2827
}
2928

3029
TIntrusivePtr<THullCtx> HullCtx;
@@ -92,7 +91,7 @@ namespace NKikimr {
9291
const auto& topology = *HullCtx->VCtx->Top; // topology we have
9392
Y_ABORT_UNLESS(topology.BelongsToSubgroup(self, CurKey.Hash())); // check that blob belongs to subgroup
9493

95-
if (!Filter->Check(CurKey, CurIt.GetMemRec(), CurIt.GetMemRecsMerged(), HullCtx->AllowKeepFlags)) {
94+
if (!Filter->Check(CurKey, CurIt.GetMemRec(), HullCtx->AllowKeepFlags)) {
9695
// filter check returned false
9796
return false;
9897
}

‎ydb/core/blobstorage/vdisk/defrag/defrag_search.h‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ namespace NKikimr {
6969
TDataMerger Merger;
7070
TKeyLogoBlob Key;
7171
TMemRecLogoBlob MemRec;
72-
ui32 NumMemRecsMerged;
7372

7473
public:
7574
TDefragScanner(THullDsSnap&& fullSnap)
@@ -99,13 +98,11 @@ namespace NKikimr {
9998
void AddFromFresh(const TMemRecLogoBlob& memRec, const TRope* /*data*/, const TKeyLogoBlob& key, ui64 lsn) {
10099
Update(memRec, nullptr, lsn);
101100
MemRec.Merge(memRec, key);
102-
++NumMemRecsMerged;
103101
}
104102

105103
void AddFromSegment(const TMemRecLogoBlob& memRec, const TDiskPart *outbound, const TKeyLogoBlob& key, ui64 circaLsn) {
106104
Update(memRec, outbound, circaLsn);
107105
MemRec.Merge(memRec, key);
108-
++NumMemRecsMerged;
109106
}
110107

111108
static constexpr bool HaveToMergeData() { return false; }
@@ -114,13 +111,12 @@ namespace NKikimr {
114111
void Start(const TKeyLogoBlob& key) {
115112
Key = key;
116113
MemRec = {};
117-
NumMemRecsMerged = 0;
118114
}
119115

120116
void Finish() {
121117
if (!Merger.Empty()) {
122118
Y_ABORT_UNLESS(!Merger.HasSmallBlobs());
123-
NGc::TKeepStatus status = Barriers->Keep(Key, MemRec, NumMemRecsMerged, AllowKeepFlags, true /*allowGarbageCollection*/);
119+
NGc::TKeepStatus status = Barriers->Keep(Key, MemRec, {}, AllowKeepFlags, true /*allowGarbageCollection*/);
124120
const auto& hugeMerger = Merger.GetHugeBlobMerger();
125121
const auto& local = MemRec.GetIngress().LocalParts(GType);
126122
ui8 partIdx = local.FirstPosition();

‎ydb/core/blobstorage/vdisk/hulldb/barriers/barriers_essence.cpp‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ namespace NKikimr {
6060

6161
NGc::TKeepStatus TBarriersEssence::KeepLogoBlob(const TLogoBlobID &id,
6262
const TIngress &ingress,
63-
const ui32 recsMerged,
63+
TKeepFlagStat keepFlagStat,
6464
const bool allowKeepFlags,
6565
bool allowGarbageCollection) const
6666
{
@@ -88,17 +88,14 @@ namespace NKikimr {
8888
// flags says us to keep the record?
8989
const bool keepByFlags = ingress.KeepUnconditionally(IngressMode);
9090

91-
// is item is spread over multiple ssts?
92-
const bool itemIsSpreadOverMultipleSsts = recsMerged > 1;
93-
9491
// check if we have to keep data associated with this blob
9592
const bool keepData = (keepBySoftBarrier || keepByFlags) && keepByHardBarrier;
9693

9794
// check if we have to keep index data; when item is stored in multiple SSTables, we can't just drop one
9895
// index and keep others as this index may contain Keep flag vital for blob consistency -- in case when
9996
// we drop such record, we will lose keep flag and item will be occasionally on next compaction; anyway,
10097
// if hard barrier tells us to drop this item, we drop it unconditinally
101-
const bool spreadFactor = allowKeepFlags && itemIsSpreadOverMultipleSsts;
98+
const bool spreadFactor = allowKeepFlags && keepFlagStat.Needed;
10299
const bool keepIndex = (keepData || spreadFactor) && keepByHardBarrier;
103100

104101
return NGc::TKeepStatus(keepIndex, keepData, keepBySoftBarrier && keepByHardBarrier);

‎ydb/core/blobstorage/vdisk/hulldb/barriers/barriers_essence.h‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,23 @@ namespace NKikimr {
1515

1616
namespace NGcOpt {
1717

18+
struct TKeepFlagStat {
19+
bool Needed = false;
20+
21+
TKeepFlagStat() = default;
22+
23+
template<typename TKey, typename TMemRec>
24+
TKeepFlagStat(const TRecordMergerBase<TKey, TMemRec>& subs, const TRecordMergerBase<TKey, TMemRec>& whole)
25+
: Needed(subs.GetNumDoNotKeepFlags() == whole.GetNumDoNotKeepFlags() && // DoNotKeep flag only in this record
26+
subs.GetNumKeepFlags() < whole.GetNumKeepFlags()) // and Keep flag somewhere else
27+
{
28+
// Needed is set to true when we are going to compact this record, but this is the only metadata record that
29+
// contains DoNotKeep flag for the blob; in this case we have to keep the record without any data to prevent
30+
// DoNotKeep from vanishing; this flag is used only when the blob is deletable (i.e. has no flags at all, or
31+
// has both Keep and DoNotKeep, and also it is beyond the soft barrier)
32+
}
33+
};
34+
1835
//////////////////////////////////////////////////////////////////////////////////////////
1936
// TBarriersEssence
2037
//////////////////////////////////////////////////////////////////////////////////////////
@@ -34,17 +51,15 @@ namespace NGcOpt {
3451

3552
NGc::TKeepStatus Keep(const TKeyLogoBlob &key,
3653
const TMemRecLogoBlob &memRec,
37-
ui32 recsMerged,
54+
TKeepFlagStat keepFlagStat,
3855
bool allowKeepFlags,
3956
bool allowGarbageCollection) const {
40-
const TIngress ingress = memRec.GetIngress();
41-
Y_DEBUG_ABORT_UNLESS(recsMerged >= 1);
42-
return KeepLogoBlob(key.LogoBlobID(), ingress, recsMerged, allowKeepFlags, allowGarbageCollection);
57+
return KeepLogoBlob(key.LogoBlobID(), memRec.GetIngress(), keepFlagStat, allowKeepFlags, allowGarbageCollection);
4358
}
4459

4560
NGc::TKeepStatus Keep(const TKeyBlock& /*key*/,
4661
const TMemRecBlock& /*memRec*/,
47-
ui32 /*recsMerged*/,
62+
TKeepFlagStat /*keepFlagStat*/,
4863
bool /*allowKeepFlags*/,
4964
bool /*allowGarbageCollection*/) const {
5065
// NOTE: We never delete block records, we only merge them. Merge rules are
@@ -55,7 +70,7 @@ namespace NGcOpt {
5570

5671
NGc::TKeepStatus Keep(const TKeyBarrier& key,
5772
const TMemRecBarrier& /*memRec*/,
58-
ui32 /*recsMerged*/,
73+
TKeepFlagStat /*keepFlagStat*/,
5974
bool /*allowKeepFlags*/,
6075
bool /*allowGarbageCollection*/) const {
6176
return KeepBarrier(key);
@@ -77,7 +92,7 @@ namespace NGcOpt {
7792
NGc::TKeepStatus KeepBarrier(const TKeyBarrier &key) const;
7893
NGc::TKeepStatus KeepLogoBlob(const TLogoBlobID &id,
7994
const TIngress &ingress,
80-
const ui32 recsMerged,
95+
TKeepFlagStat keepFlagStat,
8196
const bool allowKeepFlags,
8297
bool allowGarbageCollection) const;
8398
};

‎ydb/core/blobstorage/vdisk/hulldb/blobstorage_hullgcmap.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,7 @@ namespace NKikimr {
140140
Y_UNUSED(subsMerger);
141141
bool allowKeepFlags = HullCtx->AllowKeepFlags;
142142
NGc::TKeepStatus keep = barriersEssence->Keep(dbIt.GetCurKey(), dbMerger.GetMemRec(),
143-
dbMerger.GetMemRecsMerged(), allowKeepFlags,
144-
AllowGarbageCollection);
143+
{subsMerger, dbMerger}, allowKeepFlags, AllowGarbageCollection);
145144
Stat.Update(dbIt.GetCurKey(), keep);
146145
if (keep.KeepIndex) {
147146
IndexKeepMap.Set(Stat.ItemsTotal - 1);

‎ydb/core/blobstorage/vdisk/hulldb/compstrat/hulldb_compstrat_ratio.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ namespace NKikimr {
165165
bool allowKeepFlags = HullCtx->AllowKeepFlags;
166166
NGc::TKeepStatus keep = BarriersEssence->Keep(dbIt.GetCurKey(),
167167
dbMerger.GetMemRec(),
168-
dbMerger.GetMemRecsMerged(),
168+
{subsMerger, dbMerger},
169169
allowKeepFlags,
170170
AllowGarbageCollection);
171171
if (keep.KeepIndex) {

‎ydb/core/blobstorage/vdisk/hulldb/generic/blobstorage_hullrecmerger.h‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,31 +23,41 @@ namespace NKikimr {
2323
return MergeData;
2424
}
2525

26-
ui32 GetMemRecsMerged() const {
27-
return MemRecsMerged;
28-
}
26+
ui32 GetMemRecsMerged() const { return MemRecsMerged; }
27+
ui32 GetNumKeepFlags() const { return NumKeepFlags; }
28+
ui32 GetNumDoNotKeepFlags() const { return NumDoNotKeepFlags >> 1; }
2929

3030
protected:
3131
const TBlobStorageGroupType GType;
3232
TMemRec MemRec;
33-
ui32 MemRecsMerged; // number of items that took part in a merge for the current key
33+
ui32 MemRecsMerged = 0; // number of items that took part in a merge for the current key
34+
ui32 NumKeepFlags = 0;
35+
ui32 NumDoNotKeepFlags = 0;
3436
bool Finished;
3537
const bool MergeData;
3638

3739
TRecordMergerBase(const TBlobStorageGroupType &gtype, bool mergeData)
3840
: GType(gtype)
3941
, MemRec()
40-
, MemRecsMerged(0)
4142
, Finished(false)
4243
, MergeData(mergeData)
4344
{}
4445

4546
void Clear() {
4647
MemRecsMerged = 0;
48+
NumKeepFlags = 0;
49+
NumDoNotKeepFlags = 0;
4750
Finished = false;
4851
}
4952

5053
void AddBasic(const TMemRec &memRec, const TKey &key) {
54+
if constexpr (std::is_same_v<TMemRec, TMemRecLogoBlob>) {
55+
const int mode = memRec.GetIngress().GetCollectMode(TIngress::IngressMode(GType));
56+
static_assert(CollectModeKeep == 1);
57+
static_assert(CollectModeDoNotKeep == 2);
58+
NumKeepFlags += mode & CollectModeKeep;
59+
NumDoNotKeepFlags += mode & CollectModeDoNotKeep;
60+
}
5161
if (MemRecsMerged == 0) {
5262
MemRec = memRec;
5363
MemRec.SetNoBlob();

‎ydb/core/blobstorage/vdisk/query/query_extr.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ namespace NKikimr {
6666

6767
template<typename TMerger>
6868
bool IsBlobDeleted(const TLogoBlobID &id, const TMerger &merger) {
69-
const auto &status = BarriersEssence->Keep(id, merger.GetMemRec(), merger.GetMemRecsMerged(),
70-
QueryCtx->HullCtx->AllowKeepFlags, true /*allowGarbageCollection*/);
69+
const auto &status = BarriersEssence->Keep(id, merger.GetMemRec(), {}, QueryCtx->HullCtx->AllowKeepFlags,
70+
true /*allowGarbageCollection*/);
7171
return !status.KeepData;
7272
}
7373

‎ydb/core/blobstorage/vdisk/query/query_range.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,8 @@ namespace NKikimr {
123123

124124
template<typename TMerger>
125125
void AddIndexOnly(const TLogoBlobID &logoBlobId, const TMerger &merger) {
126-
const auto &status = BarriersEssence->Keep(logoBlobId, merger.GetMemRec(), merger.GetMemRecsMerged(),
127-
QueryCtx->HullCtx->AllowKeepFlags, true /*allowGarbageCollection*/);
126+
const auto &status = BarriersEssence->Keep(logoBlobId, merger.GetMemRec(), {},
127+
QueryCtx->HullCtx->AllowKeepFlags, true /*allowGarbageCollection*/);
128128
if (status.KeepData) {
129129
const TIngress &ingress = merger.GetMemRec().GetIngress();
130130
ui64 ingr = ingress.Raw();

0 commit comments

Comments
 (0)