Skip to content

Commit

Permalink
[BugFix] only create IO Entry when enable IOProfiler (backport #52756) (
Browse files Browse the repository at this point in the history 
#52822)

Signed-off-by: stdpain <34912776+stdpain@users.noreply.github.com>
Co-authored-by: stdpain <34912776+stdpain@users.noreply.github.com>
  • Loading branch information
@mergify @stdpain
mergify[bot] and stdpain authored Nov 12, 2024
1 parent 89eb395 commit 2070e49
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 55 deletions.
21 changes: 9 additions & 12 deletions be/src/io/io_profiler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

#include "io_profiler.h"

#include <cstdint>
#include <memory>
#include <mutex>
#include <thread>
#include <unordered_set>
Expand Down Expand Up @@ -98,13 +100,8 @@ struct IOStatEntry {
}
};

class IOStatEntryHash {
public:
size_t operator()(const IOStatEntry& entry) const { return std::hash<uint64_t>()(entry.id); }
};

static std::mutex _io_stats_mutex;
static std::unordered_set<IOStatEntry, IOStatEntryHash> _io_stats;
static std::unordered_map<uint64_t, std::unique_ptr<IOStatEntry>> _io_stats;

bool IOProfiler::is_empty() {
return _io_stats.empty();
Expand All @@ -126,16 +123,16 @@ void IOProfiler::reset() {
thread_local IOStatEntry* current_io_stat = nullptr;

void IOProfiler::set_context(uint32_t tag, uint64_t tablet_id) {
if (tablet_id == 0) {
if (tablet_id == 0 || _context_io_mode == IOMode::IOMODE_NONE) {
return;
}
uint64_t key = ((uint64_t)tag << 48UL) | tablet_id;
std::lock_guard<std::mutex> l(_io_stats_mutex);
auto it = _io_stats.find(IOStatEntry{key});
auto it = _io_stats.find(key);
if (it == _io_stats.end()) {
it = _io_stats.emplace(key).first;
it = _io_stats.emplace(key, std::make_unique<IOStatEntry>(key)).first;
}
current_io_stat = const_cast<IOStatEntry*>(&(*it));
current_io_stat = it->second.get();
}

IOStatEntry* IOProfiler::get_context() {
Expand Down Expand Up @@ -244,9 +241,9 @@ StatusOr<std::vector<std::string>> IOProfiler::get_topn_stats(size_t n,
}
stats.reserve(_io_stats.size());
for (auto& it : _io_stats) {
auto v = func(it);
auto v = func(*it.second);
if (v > 0) {
stats.emplace_back(v, &it);
stats.emplace_back(v, it.second.get());
}
}
std::sort(stats.begin(), stats.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
Expand Down
2 changes: 1 addition & 1 deletion be/src/io/io_profiler.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

namespace starrocks {

class IOStatEntry;
struct IOStatEntry;

class IOProfiler {
public:
Expand Down
98 changes: 56 additions & 42 deletions be/test/io/io_profiler_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ namespace starrocks {
ASSERT_EQ(expect.sync_time_ns, actual.sync_time_ns)

TEST(IOProfilerTest, test_tls_io) {
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 1);
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_ALL));
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 6);
// the init io stat may be not 0, because other test cases maybe have updated the stat
auto expect = scope.current_scoped_tls_io();

Expand Down Expand Up @@ -73,48 +74,61 @@ TEST(IOProfilerTest, test_tls_io) {
ADD_SYNC_IO_STAT(expect, 9883);
auto actual5 = scope.current_scoped_tls_io();
ASSERT_IO_STAT_EQ(expect, actual5);
IOProfiler::stop();
}

TEST(IOProfilerTest, test_context_io) {
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 2);
auto expect = IOProfiler::IOStat{0, 0, 0, 0, 0, 0};

// io mode is IOMODE_NONE
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());
IOProfiler::add_read(1, 1000);
IOProfiler::add_write(1, 1000);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());

// io mode is IOMODE_READ
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_READ));
ASSERT_EQ(IOProfiler::IOMode::IOMODE_READ, IOProfiler::get_context_io_mode());
IOProfiler::add_read(2, 2000);
ADD_READ_IO_STAT(expect, 2, 0);
IOProfiler::add_write(2, 2000);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
IOProfiler::stop();
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());

// io mode is IOMODE_WRITE
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_WRITE));
ASSERT_EQ(IOProfiler::IOMode::IOMODE_WRITE, IOProfiler::get_context_io_mode());
IOProfiler::add_read(3, 3000);
IOProfiler::add_write(3, 3000);
ADD_WRITE_IO_STAT(expect, 3, 0);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
IOProfiler::stop();
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());

// io mode is IOMODE_ALL
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_ALL));
ASSERT_EQ(IOProfiler::IOMode::IOMODE_ALL, IOProfiler::get_context_io_mode());
IOProfiler::add_read(4, 4000);
ADD_READ_IO_STAT(expect, 4, 0);
IOProfiler::add_write(4, 4000);
ADD_WRITE_IO_STAT(expect, 4, 0);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
IOProfiler::stop();
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());
{
auto expect = IOProfiler::IOStat{0, 0, 0, 0, 0, 0};
// io mode is IOMODE_NONE
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 1);
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());
IOProfiler::add_read(1, 1000);
IOProfiler::add_write(1, 1000);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
}
{
auto expect = IOProfiler::IOStat{0, 0, 0, 0, 0, 0};
// io mode is IOMODE_READ
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_READ));
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 2);

ASSERT_EQ(IOProfiler::IOMode::IOMODE_READ, IOProfiler::get_context_io_mode());
IOProfiler::add_read(2, 2000);
ADD_READ_IO_STAT(expect, 2, 0);
IOProfiler::add_write(2, 2000);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
IOProfiler::stop();
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());
}
{
auto expect = IOProfiler::IOStat{0, 0, 0, 0, 0, 0};
// io mode is IOMODE_WRITE
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_WRITE));
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 3);

ASSERT_EQ(IOProfiler::IOMode::IOMODE_WRITE, IOProfiler::get_context_io_mode());
IOProfiler::add_read(3, 3000);
IOProfiler::add_write(3, 3000);
ADD_WRITE_IO_STAT(expect, 3, 0);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
IOProfiler::stop();
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());
}
{
auto expect = IOProfiler::IOStat{0, 0, 0, 0, 0, 0};
// io mode is IOMODE_ALL
ASSERT_OK(IOProfiler::start(IOProfiler::IOMode::IOMODE_ALL));
auto scope = IOProfiler::scope(IOProfiler::TAG_LOAD, 4);

ASSERT_EQ(IOProfiler::IOMode::IOMODE_ALL, IOProfiler::get_context_io_mode());
IOProfiler::add_read(4, 4000);
ADD_READ_IO_STAT(expect, 4, 0);
IOProfiler::add_write(4, 4000);
ADD_WRITE_IO_STAT(expect, 4, 0);
ASSERT_IO_STAT_EQ(expect, scope.current_context_io());
IOProfiler::stop();
ASSERT_EQ(IOProfiler::IOMode::IOMODE_NONE, IOProfiler::get_context_io_mode());
}
}

} // namespace starrocks
} // namespace starrocks

0 comments on commit 2070e49

Please sign in to comment.