Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make mempurge a background process (equivalent to in-memory compaction). #8505

Closed
Closed
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
00fa140
Created FlushJob Mempurge funciton
bjlemaire Jul 6, 2021
491d6d6
Created new mempurge function as a backgroudn task. Need to create ne…
bjlemaire Jul 6, 2021
346052e
Populate MemPurgeV2 with code to handle more than one immutable memta…
bjlemaire Jul 6, 2021
9d41c33
Remove first prototype of memprueg that happend during memtable switc…
bjlemaire Jul 7, 2021
14cc37b
Fix deadlock situation. Pass first 2 tests. Still flushes regular tab…
bjlemaire Jul 8, 2021
46094f1
Add sync points for tests. Successful test 1, 2, and 3, but more work…
bjlemaire Jul 8, 2021
ae89742
Remodeled unit test 1 and 2, mempurge passes these 2 tests. Need for …
bjlemaire Jul 8, 2021
0819b34
Update test 3. Now mempurge passes all 3 tests for get put delete ran…
bjlemaire Jul 8, 2021
bd16dea
Fix typo in mempurge comment.
bjlemaire Jul 8, 2021
becf067
cast uint64 to uint32 to avoid implicit conversion loss
bjlemaire Jul 8, 2021
40d5749
Run make format.
bjlemaire Jul 8, 2021
64f8b01
Fix column ID of newly created memtables in mempurge.
bjlemaire Jul 8, 2021
cefd967
Fix potential memory leak pointed by clang_analyzer.
bjlemaire Jul 8, 2021
cc4a8a0
Fix memory leak by correctly cleaning up mempurge output memtables fl…
bjlemaire Jul 9, 2021
b57b7f1
Remove assert statment used for debugging.
bjlemaire Jul 9, 2021
eb9d9b2
Fix overshadow and increase number overwrites in mempurge cfilter test.
bjlemaire Jul 9, 2021
b8f034b
This one is called: always test locally before pushing.
bjlemaire Jul 9, 2021
b2d9334
Speed up unit tests. CLean flush of all column families upon DB close…
bjlemaire Jul 9, 2021
30acdd8
Fix ubsan error: replace &memtables[0] by memtables.data() to avoid u…
bjlemaire Jul 9, 2021
e034b09
Change strategy: no mempurge if memtable filled at more than 60perc c…
bjlemaire Jul 9, 2021
c4a6803
Address PR comments.
bjlemaire Jul 9, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 6 additions & 15 deletions db/column_family.cc
Original file line number Diff line number Diff line change
Expand Up @@ -443,26 +443,17 @@ bool SuperVersion::Unref() {
return previous_refs == 1;
}

void SuperVersion::Cleanup(const bool noImmMemoryContribution) {
void SuperVersion::Cleanup() {
assert(refs.load(std::memory_order_relaxed) == 0);
// Since this SuperVersion object is being deleted,
// decrement reference to the immutable MemtableList
// this SV object was pointing to.
imm->Unref(&to_delete);
MemTable* m = mem->Unref();
if (m != nullptr) {
// Typically, if the m memtable was not made
// immutable, and therefore was not added to the
// imm list, it does not contribute to the imm
// memory footprint (and actually is not part of
// the 'imm' MemtableList at all).
// At the moment, noImmMemoryContribution is only
// used by the experimental 'MemPurge' prototype.
if (!noImmMemoryContribution) {
pdillinger marked this conversation as resolved.
Show resolved Hide resolved
auto* memory_usage = current->cfd()->imm()->current_memory_usage();
assert(*memory_usage >= m->ApproximateMemoryUsage());
*memory_usage -= m->ApproximateMemoryUsage();
}
auto* memory_usage = current->cfd()->imm()->current_memory_usage();
assert(*memory_usage >= m->ApproximateMemoryUsage());
*memory_usage -= m->ApproximateMemoryUsage();
to_delete.push_back(m);
}
current->Unref();
Expand Down Expand Up @@ -1272,7 +1263,7 @@ void ColumnFamilyData::InstallSuperVersion(

void ColumnFamilyData::InstallSuperVersion(
SuperVersionContext* sv_context, InstrumentedMutex* db_mutex,
const MutableCFOptions& mutable_cf_options, bool noImmMemoryContribution) {
pdillinger marked this conversation as resolved.
Show resolved Hide resolved
const MutableCFOptions& mutable_cf_options) {
SuperVersion* new_superversion = sv_context->new_superversion.release();
new_superversion->db_mutex = db_mutex;
new_superversion->mutable_cf_options = mutable_cf_options;
Expand Down Expand Up @@ -1302,7 +1293,7 @@ void ColumnFamilyData::InstallSuperVersion(
new_superversion->write_stall_condition, GetName(), ioptions());
}
if (old_superversion->Unref()) {
old_superversion->Cleanup(noImmMemoryContribution);
pdillinger marked this conversation as resolved.
Show resolved Hide resolved
old_superversion->Cleanup();
sv_context->superversions_to_free.push_back(old_superversion);
}
}
Expand Down
9 changes: 3 additions & 6 deletions db/column_family.h
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,7 @@ struct SuperVersion {
// Cleanup unrefs mem, imm and current. Also, it stores all memtables
// that needs to be deleted in to_delete vector. Unrefing those
// objects needs to be done in the mutex
// The 'noImmMemoryContribution' is set to true if the memtable being
// dereferenced in this SuperVersion was not added to the Immutable
// memtable list.
void Cleanup(bool noImmMemoryContribution = false);
pdillinger marked this conversation as resolved.
Show resolved Hide resolved
void Cleanup();
void Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
MemTableListVersion* new_imm, Version* new_current);

Expand Down Expand Up @@ -457,8 +454,7 @@ class ColumnFamilyData {
// IMPORTANT: Only call this from DBImpl::InstallSuperVersion()
void InstallSuperVersion(SuperVersionContext* sv_context,
InstrumentedMutex* db_mutex,
const MutableCFOptions& mutable_cf_options,
bool noImmMemoryContribution = false);
pdillinger marked this conversation as resolved.
Show resolved Hide resolved
const MutableCFOptions& mutable_cf_options);
void InstallSuperVersion(SuperVersionContext* sv_context,
InstrumentedMutex* db_mutex);

Expand Down Expand Up @@ -524,6 +520,7 @@ class ColumnFamilyData {
}

ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); }
WriteBufferManager* write_buffer_mgr() { return write_buffer_manager_; }

private:
friend class ColumnFamilySet;
Expand Down
Loading