Skip to content

Commit

Permalink
new write stall stats
Browse files Browse the repository at this point in the history
  • Loading branch information
@hx235
hx235 committed Mar 18, 2023
1 parent eac6b6d commit a1fa82a
Show file tree
Hide file tree
Showing 17 changed files with 635 additions and 98 deletions.
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,7 @@ set(SOURCES
db/write_batch.cc
db/write_batch_base.cc
db/write_controller.cc
db/write_stall_stats.cc
db/write_thread.cc
env/composite_env.cc
env/env.cc
Expand Down
2 changes: 2 additions & 0 deletions HISTORY.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
* Add statistics rocksdb.secondary.cache.filter.hits, rocksdb.secondary.cache.index.hits, and rocksdb.secondary.cache.filter.hits
* Added a new PerfContext counter `internal_merge_point_lookup_count` which tracks the number of Merge operands applied while serving point lookup queries.
* Add new statistics rocksdb.table.open.prefetch.tail.read.bytes, rocksdb.table.open.prefetch.tail.{miss|hit}
* Add new db properties `rocksdb.cf-write-stall-stats`, `rocksdb.db-write-stall-stats`and APIs to examine them in a structured way.
In particular, users of `GetMapProperty()` with property `kCFWriteStallStats`/`kDBWriteStallStats` can now use the functions in `WriteStallStatsMapKeys` to find stats in the map.

## 8.0.0 (02/19/2023)
### Behavior changes
Expand Down
2 changes: 2 additions & 0 deletions TARGETS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
"db/write_batch.cc",
"db/write_batch_base.cc",
"db/write_controller.cc",
"db/write_stall_stats.cc",
"db/write_thread.cc",
"env/composite_env.cc",
"env/env.cc",
Expand Down Expand Up @@ -449,6 +450,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
"db/write_batch.cc",
"db/write_batch_base.cc",
"db/write_controller.cc",
"db/write_stall_stats.cc",
"db/write_thread.cc",
"env/composite_env.cc",
"env/env.cc",
Expand Down
15 changes: 8 additions & 7 deletions db/column_family.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -869,7 +869,7 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
}
} // anonymous namespace

std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
std::pair<WriteStallCondition, WriteStallCause>
ColumnFamilyData::GetWriteStallConditionAndCause(
int num_unflushed_memtables, int num_l0_files,
uint64_t num_compaction_needed_bytes,
Expand Down Expand Up @@ -942,7 +942,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_STOPS, 1);
if (compaction_picker_->IsLevel0CompactionInProgress()) {
internal_stats_->AddCFStats(
InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_STOPS, 1);
InternalStats::L0_FILE_COUNT_LIMIT_STOPS_WITH_ONGOING_COMPACTION,
1);
}
ROCKS_LOG_WARN(ioptions_.logger,
"[%s] Stopping writes because we have %d level-0 files",
Expand All @@ -963,7 +964,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
SetupDelay(write_controller, compaction_needed_bytes,
prev_compaction_needed_bytes_, was_stopped,
mutable_cf_options.disable_auto_compactions);
internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_SLOWDOWNS, 1);
internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_DELAYS, 1);
ROCKS_LOG_WARN(
ioptions_.logger,
"[%s] Stalling writes because we have %d immutable memtables "
Expand All @@ -981,11 +982,11 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
SetupDelay(write_controller, compaction_needed_bytes,
prev_compaction_needed_bytes_, was_stopped || near_stop,
mutable_cf_options.disable_auto_compactions);
internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_SLOWDOWNS,
1);
internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_DELAYS, 1);
if (compaction_picker_->IsLevel0CompactionInProgress()) {
internal_stats_->AddCFStats(
InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS, 1);
InternalStats::L0_FILE_COUNT_LIMIT_DELAYS_WITH_ONGOING_COMPACTION,
1);
}
ROCKS_LOG_WARN(ioptions_.logger,
"[%s] Stalling writes because we have %d level-0 files "
Expand All @@ -1011,7 +1012,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
prev_compaction_needed_bytes_, was_stopped || near_stop,
mutable_cf_options.disable_auto_compactions);
internal_stats_->AddCFStats(
InternalStats::PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS, 1);
InternalStats::PENDING_COMPACTION_BYTES_LIMIT_DELAYS, 1);
ROCKS_LOG_WARN(
ioptions_.logger,
"[%s] Stalling writes because of estimated pending compaction "
Expand Down
6 changes: 0 additions & 6 deletions db/column_family.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,6 @@ class ColumnFamilyData {
bool queued_for_flush() { return queued_for_flush_; }
bool queued_for_compaction() { return queued_for_compaction_; }

enum class WriteStallCause {
kNone,
kMemtableLimit,
kL0FileCountLimit,
kPendingCompactionBytes,
};
static std::pair<WriteStallCondition, WriteStallCause>
GetWriteStallConditionAndCause(
int num_unflushed_memtables, int num_l0_files,
Expand Down
2 changes: 2 additions & 0 deletions db/db_impl/db_impl_write.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1211,6 +1211,8 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
// exceeded at this point so no new write (including current one) will go
// through until memory usage is decreased.
if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldStall())) {
default_cf_internal_stats_->AddDBStats(
InternalStats::kIntStatsWriteBufferManagerLimitStopCounts, 1);
if (write_options.no_slowdown) {
status = Status::Incomplete("Write stall");
} else {
Expand Down
108 changes: 108 additions & 0 deletions db/db_properties_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2106,6 +2106,114 @@ TEST_F(DBPropertiesTest, GetMapPropertyBlockCacheEntryStats) {
ASSERT_EQ(3 * kNumCacheEntryRoles + 4, values.size());
}

TEST_F(DBPropertiesTest, GetMapPropertyWriteStallStats) {
Options options = CurrentOptions();
CreateAndReopenWithCF({"heavy_write_cf"}, options);

for (auto test_cause : {WriteStallCause::kWriteBufferManagerLimit,
WriteStallCause::kMemtableLimit}) {
if (test_cause == WriteStallCause::kWriteBufferManagerLimit) {
options.write_buffer_manager.reset(
new WriteBufferManager(100000, nullptr, true));
} else if (test_cause == WriteStallCause::kMemtableLimit) {
options.max_write_buffer_number = 2;
options.disable_auto_compactions = true;
}
ReopenWithColumnFamilies({"default", "heavy_write_cf"}, options);

// Assert initial write stall stats are all 0
std::map<std::string, std::string> db_values;
ASSERT_TRUE(dbfull()->GetMapProperty(DB::Properties::kDBWriteStallStats,
&db_values));
ASSERT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::TotalStop()]), 0);
ASSERT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::TotalDelay()]), 0);

for (int cf = 0; cf <= 1; ++cf) {
std::map<std::string, std::string> cf_values;
ASSERT_TRUE(dbfull()->GetMapProperty(
handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values));
ASSERT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStop()]), 0);
ASSERT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelay()]), 0);
}

// Pause flush thread to help coerce write stall
std::unique_ptr<test::SleepingBackgroundTask> sleeping_task(
new test::SleepingBackgroundTask());
env_->SetBackgroundThreads(1, Env::HIGH);
env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
sleeping_task.get(), Env::Priority::HIGH);
sleeping_task->WaitUntilSleeping();

// Coerce write stall
if (test_cause == WriteStallCause::kWriteBufferManagerLimit) {
ASSERT_OK(dbfull()->Put(
WriteOptions(), handles_[1], Key(1),
DummyString(options.write_buffer_manager->buffer_size())));

WriteOptions wo;
wo.no_slowdown = true;
Status s = dbfull()->Put(
wo, handles_[1], Key(2),
DummyString(options.write_buffer_manager->buffer_size()));
ASSERT_TRUE(s.IsIncomplete());
ASSERT_TRUE(s.ToString().find("Write stall") != std::string::npos);
} else if (test_cause == WriteStallCause::kMemtableLimit) {
FlushOptions fo;
fo.allow_write_stall = true;
fo.wait = false;

ASSERT_OK(
dbfull()->Put(WriteOptions(), handles_[1], Key(1), DummyString(1)));
ASSERT_OK(dbfull()->Flush(fo, handles_[1]));

ASSERT_OK(
dbfull()->Put(WriteOptions(), handles_[1], Key(2), DummyString(1)));
ASSERT_OK(dbfull()->Flush(fo, handles_[1]));
}

if (test_cause == WriteStallCause::kWriteBufferManagerLimit) {
db_values.clear();
EXPECT_TRUE(dbfull()->GetMapProperty(DB::Properties::kDBWriteStallStats,
&db_values));
EXPECT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::TotalStop()]), 1);
EXPECT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::TotalDelay()]), 0);
// `WriteStallCause::kWriteBufferManagerLimit` should not result in any
// CF-scope write stall stats changes
for (int cf = 0; cf <= 1; ++cf) {
std::map<std::string, std::string> cf_values;
EXPECT_TRUE(dbfull()->GetMapProperty(
handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values));
EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStop()]), 0);
EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelay()]),
0);
}
} else if (test_cause == WriteStallCause::kMemtableLimit) {
for (int cf = 0; cf <= 1; ++cf) {
std::map<std::string, std::string> cf_values;
EXPECT_TRUE(dbfull()->GetMapProperty(
handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values));
EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStop()]),
cf == 1 ? 1 : 0);
EXPECT_EQ(
std::stoi(cf_values[WriteStallStatsMapKeys::CauseConditionCount(
WriteStallCause::kMemtableLimit,
WriteStallCondition::kStopped)]),
cf == 1 ? 1 : 0);
EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelay()]),
0);
EXPECT_EQ(
std::stoi(cf_values[WriteStallStatsMapKeys::CauseConditionCount(
WriteStallCause::kMemtableLimit,
WriteStallCondition::kDelayed)]),
0);
}
}

sleeping_task->WakeUp();
sleeping_task->WaitUntilDone();
}
}

namespace {
std::string PopMetaIndexKey(InternalIterator* meta_iter) {
Status s = meta_iter->status();
Expand Down
Loading

0 comments on commit a1fa82a

Please sign in to comment.