Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions benchmarks/json_output_benchmark.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,9 @@ double runBenchmarkIteration(

// Finalize the trace
const Config config;
std::unordered_map<std::string, std::vector<std::string>> finalMetadata;
const int64_t endTime =
activities.empty() ? span.endTime : activities.back().endTime;
logger.finalizeTrace(config, nullptr, endTime, finalMetadata);
logger.finalizeTrace(config, nullptr, endTime);
}

auto end = std::chrono::steady_clock::now();
Expand Down
5 changes: 1 addition & 4 deletions libkineto/include/output_base.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,7 @@ class ActivityLogger {

virtual void finalizeMemoryTrace(const std::string&, const Config&) = 0;

virtual void finalizeTrace(const Config& config,
std::unique_ptr<ActivityBuffers> buffers,
int64_t endTime,
std::unordered_map<std::string, std::vector<std::string>>& metadata) = 0;
virtual void finalizeTrace(const Config& config, std::unique_ptr<ActivityBuffers> buffers, int64_t endTime) = 0;

protected:
ActivityLogger() = default;
Expand Down
34 changes: 1 addition & 33 deletions libkineto/src/GenericActivityProfiler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -478,12 +478,6 @@ void GenericActivityProfiler::configure(
// Ensure we're starting in a clean state
resetTraceData();

#if !USE_GOOGLE_LOG
// Add a LoggerObserverCollector to collect all logs during the trace.
loggerCollectorMetadata_ = std::make_unique<LoggerCollector>();
Logger::addLoggerObserver(loggerCollectorMetadata_.get());
#endif // !USE_GOOGLE_LOG

derivedConfig_.reset();
derivedConfig_ = std::make_unique<ConfigDerivedState>(*config_);

Expand Down Expand Up @@ -736,30 +730,7 @@ void GenericActivityProfiler::finalizeTrace(
}
}

// Logger Metadata contains a map of LOGs collected in Kineto
// logger_level -> List of log lines
// This will be added into the trace as metadata.
std::unordered_map<std::string, std::vector<std::string>> loggerMD =
getLoggerMetadata();
logger.finalizeTrace(
config, std::move(traceBuffers_), captureWindowEndTime_, loggerMD);
}

std::unordered_map<std::string, std::vector<std::string>>
GenericActivityProfiler::getLoggerMetadata() {
std::unordered_map<std::string, std::vector<std::string>> loggerMD;

#if !USE_GOOGLE_LOG
// Save logs from LoggerCollector objects into Trace metadata.
auto LoggerMDMap = loggerCollectorMetadata_->extractCollectorMetadata();
for (auto& md : LoggerMDMap) {
// Only WARNING and ERROR are included in trace metadata.
if (md.first == WARNING || md.first == ERROR) {
loggerMD[toString(md.first)] = md.second;
}
}
#endif // !USE_GOOGLE_LOG
return loggerMD;
logger.finalizeTrace(config, std::move(traceBuffers_), captureWindowEndTime_);
}

void GenericActivityProfiler::pushCorrelationId(uint64_t id) {
Expand Down Expand Up @@ -808,9 +779,6 @@ void GenericActivityProfiler::resetTraceData() {
sessions_.clear();
resourceOverheadCount_ = 0;
ecs_ = ErrorCounts{};
#if !USE_GOOGLE_LOG
Logger::removeLoggerObserver(loggerCollectorMetadata_.get());
#endif // !USE_GOOGLE_LOG
}

void GenericActivityProfiler::collectTrace(
Expand Down
8 changes: 0 additions & 8 deletions libkineto/src/GenericActivityProfiler.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@

#include "GenericTraceActivity.h"
#include "IActivityProfiler.h"
#include "LoggerCollector.h"
#include "ThreadUtil.h"
#include "TraceSpan.h"
#include "libkineto.h"
Expand Down Expand Up @@ -258,8 +257,6 @@ class GenericActivityProfiler {
profilers_.push_back(std::move(profiler));
}

std::unordered_map<std::string, std::vector<std::string>> getLoggerMetadata();

void pushCorrelationId(uint64_t id);
void popCorrelationId();

Expand Down Expand Up @@ -518,11 +515,6 @@ class GenericActivityProfiler {
uint32_t resourceOverheadCount_;

ErrorCounts ecs_;

// LoggerCollector to collect all LOGs during the trace
#if !USE_GOOGLE_LOG
std::unique_ptr<LoggerCollector> loggerCollectorMetadata_;
#endif // !USE_GOOGLE_LOG
};

} // namespace KINETO_NAMESPACE
7 changes: 0 additions & 7 deletions libkineto/src/SyncActivityProfilerHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,6 @@ std::unique_ptr<ActivityTraceInterface> SyncActivityProfilerHandler::
// Will follow up with another patch for logging URLs when ActivityTrace
// is moved.

// Logger Metadata contains a map of LOGs collected in Kineto
// logger_level -> List of log lines
// This will be added into the trace as metadata.
std::unordered_map<std::string, std::vector<std::string>> loggerMD =
profiler_.getLoggerMetadata();
logger->setLoggerMetadata(std::move(loggerMD));

profiler_.reset();
active_ = false;
return std::make_unique<ActivityTrace>(
Expand Down
48 changes: 3 additions & 45 deletions libkineto/src/output_json.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,6 @@ void sanitizeStrForJSON(std::string& value) {
std::erase(value, '\n');
}

// Free-form log strings: drop control chars, replace backslashes and double
// quotes
void sanitizeLogStrForJSON(std::string& value) {
std::erase_if(value, [](unsigned char c) { return c < 0x20; });
std::ranges::replace(value, '\\', '/');
std::ranges::replace(value, '"', '\'');
}

std::string string2hex(const std::string& str) {
std::string out;
out.reserve(str.size() * 2);
Expand Down Expand Up @@ -964,9 +956,8 @@ void ChromeTraceLogger::handleLink(
void ChromeTraceLogger::finalizeTrace(
[[maybe_unused]] const Config& config,
[[maybe_unused]] std::unique_ptr<ActivityBuffers> buffers,
int64_t endTime,
std::unordered_map<std::string, std::vector<std::string>>& metadata) {
finalizeTrace(endTime, metadata);
int64_t endTime) {
finalizeTrace(endTime);
}

void ChromeTraceLogger::addOnDemandDistMetadata() {
Expand Down Expand Up @@ -1005,9 +996,7 @@ void ChromeTraceLogger::addOnDemandDistMetadata() {
distInfo_.distInfo_present_ = true;
}

void ChromeTraceLogger::finalizeTrace(
int64_t endTime,
std::unordered_map<std::string, std::vector<std::string>>& metadata) {
void ChromeTraceLogger::finalizeTrace(int64_t endTime) {
if (!traceOf_) {
LOG(ERROR) << "Failed to write to log file!";
return;
Expand Down Expand Up @@ -1036,37 +1025,6 @@ void ChromeTraceLogger::finalizeTrace(
addOnDemandDistMetadata();
}

#if !USE_GOOGLE_LOG
for (const auto& kv : metadata) {
// Skip empty log buckets, ex. skip ERROR if its empty.
if (kv.second.empty()) {
continue;
}
std::string value = "[";
// Ex. Each metadata from logger is a list of strings, expressed in JSON
// as
// "ERROR": ["Error 1", "Error 2"],
// "WARNING": ["Warning 1", "Warning 2", "Warning 3"],
// ...
size_t mdv_count = kv.second.size();
for (auto v : kv.second) {
sanitizeLogStrForJSON(v);
value.append("\"" + v + "\"");
if (mdv_count > 1) {
value.append(",");
mdv_count--;
}
}
value.append("]");
fmt::print(
traceOf_,
R"JSON(
"{}": {},)JSON",
kv.first,
value);
}
#endif // !USE_GOOGLE_LOG

// The last entry MUST NOT end with a comma.
fmt::print(traceOf_, R"JSON("traceName": "{}" }})JSON", fileName_);

Expand Down
7 changes: 2 additions & 5 deletions libkineto/src/output_json.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,7 @@ class ChromeTraceLogger : public libkineto::ActivityLogger {
void handleTraceStart(const std::unordered_map<std::string, std::string>& metadata,
const std::string& device_properties) override;

void finalizeTrace(const Config& config,
std::unique_ptr<ActivityBuffers> buffers,
int64_t endTime,
std::unordered_map<std::string, std::vector<std::string>>& metadata) override;
void finalizeTrace(const Config& config, std::unique_ptr<ActivityBuffers> buffers, int64_t endTime) override;

void finalizeMemoryTrace(const std::string&, const Config&) override;

Expand All @@ -87,7 +84,7 @@ class ChromeTraceLogger : public libkineto::ActivityLogger {
}

protected:
void finalizeTrace(int64_t endTime, std::unordered_map<std::string, std::vector<std::string>>& metadata);
void finalizeTrace(int64_t endTime);

private:
// Create a flow event (arrow)
Expand Down
10 changes: 2 additions & 8 deletions libkineto/src/output_membuf.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,7 @@ class MemoryTraceLogger : public ActivityLogger {

void finalizeTrace([[maybe_unused]] const Config& config,
std::unique_ptr<ActivityBuffers> buffers,
int64_t endTime,
[[maybe_unused]] std::unordered_map<std::string, std::vector<std::string>>& metadata) override {
int64_t endTime) override {
buffers_ = std::move(buffers);
endTime_ = endTime;
}
Expand Down Expand Up @@ -100,11 +99,7 @@ class MemoryTraceLogger : public ActivityLogger {
logger.handleTraceSpan(cpu_trace_buffer->span);
}
// Hold on to the buffers
logger.finalizeTrace(*config_, nullptr, endTime_, loggerMetadata_);
}

void setLoggerMetadata(std::unordered_map<std::string, std::vector<std::string>>&& lmd) {
loggerMetadata_ = std::move(lmd);
logger.finalizeTrace(*config_, nullptr, endTime_);
}

void setChromeLogger(std::shared_ptr<ActivityLogger> logger) {
Expand All @@ -124,7 +119,6 @@ class MemoryTraceLogger : public ActivityLogger {
std::vector<std::pair<ResourceInfo, int64_t>> resourceInfoList_;
std::unique_ptr<ActivityBuffers> buffers_;
std::unordered_map<std::string, std::string> metadata_;
std::unordered_map<std::string, std::vector<std::string>> loggerMetadata_;
std::string device_properties_;
int64_t endTime_{0};
std::shared_ptr<ActivityLogger> chrome_logger_;
Expand Down
8 changes: 0 additions & 8 deletions libkineto/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,3 @@ target_link_libraries(PidInfoTest PRIVATE
kineto_base kineto_api
${XPU_XPUPTI_LIBRARY})
gtest_discover_tests(PidInfoTest)

# OutputJsonTest
add_executable(OutputJsonTest OutputJsonTest.cpp)
target_link_libraries(OutputJsonTest PRIVATE
gtest_main
kineto_base kineto_api
${XPU_XPUPTI_LIBRARY})
gtest_discover_tests(OutputJsonTest)
17 changes: 0 additions & 17 deletions libkineto/test/LoggerObserverTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -289,23 +289,6 @@ TEST(LoggerObserverTest, UstMacrosRespectSeverityThreshold) {
Logger::removeLoggerObserver(observer.get());
}

TEST(LoggerObserverTest, GetLoggerMetadataOnlyIncludesWarningAndError) {
GenericActivityProfiler profiler(/*cpuOnly=*/true);
profiler.configure(Config{}, {});

LOG(INFO) << InfoTestStr;
LOG(WARNING) << WarningTestStr;
LOG(ERROR) << ErrorTestStr;

const auto loggerMD = profiler.getLoggerMetadata();
EXPECT_EQ(loggerMD.size(), 2);
EXPECT_EQ(loggerMD.count("INFO"), 0);
EXPECT_EQ(loggerMD.count("WARNING"), 1);
EXPECT_EQ(loggerMD.count("ERROR"), 1);

profiler.reset();
}

#endif // !USE_GOOGLE_LOG

int main(int argc, char** argv) {
Expand Down
48 changes: 0 additions & 48 deletions libkineto/test/OutputJsonTest.cpp

This file was deleted.

6 changes: 2 additions & 4 deletions libkineto/test/RegisterLoggerFactoryTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ class MockActivityLogger : public libkineto::ActivityLogger {
void finalizeTrace(
const libkineto::Config&,
std::unique_ptr<libkineto::ActivityBuffers>,
int64_t,
std::unordered_map<std::string, std::vector<std::string>>&) override {}
int64_t) override {}
void finalizeMemoryTrace(const std::string&, const libkineto::Config&)
override {}

Expand Down Expand Up @@ -77,8 +76,7 @@ class CountingLogger : public libkineto::ActivityLogger {
void finalizeTrace(
const libkineto::Config&,
std::unique_ptr<libkineto::ActivityBuffers>,
int64_t,
std::unordered_map<std::string, std::vector<std::string>>&) override {}
int64_t) override {}
void finalizeMemoryTrace(const std::string&, const libkineto::Config&)
override {}

Expand Down
5 changes: 1 addition & 4 deletions libkineto/test/xpupti/XpuptiTestUtilities.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,7 @@ class TestActivityLogger : public KN::ActivityLogger {
void finalizeTrace(
[[maybe_unused]] const KN::Config& config,
[[maybe_unused]] std::unique_ptr<KN::ActivityBuffers> buffers,
[[maybe_unused]] int64_t endTime,
[[maybe_unused]] std::unordered_map<
std::string,
std::vector<std::string>>& metadata) override {}
[[maybe_unused]] int64_t endTime) override {}
};

std::pair<
Expand Down
Loading