Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
File renamed without changes.
19 changes: 14 additions & 5 deletions libkineto/include/AbstractConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ class AbstractConfig {
}

// Time config was created / updated
[[nodiscard]] std::chrono::time_point<std::chrono::system_clock> timestamp() const {
[[nodiscard]] std::chrono::time_point<std::chrono::system_clock> timestamp()
const {
return timestamp_;
}

Expand Down Expand Up @@ -75,23 +76,31 @@ class AbstractConfig {
//
// @param fallbackProfileStartTime Specify a fallback profile start timestamp
// in case it was never specified by the client
virtual void validate(const std::chrono::time_point<std::chrono::system_clock>& fallbackProfileStartTime) = 0;
virtual void validate(
const std::chrono::time_point<std::chrono::system_clock>&
fallbackProfileStartTime) = 0;

// TODO: Separate out each profiler type into features?
virtual void printActivityProfilerConfig(std::ostream& s) const;
virtual void setActivityDependentConfig();

// Helpers for use in handleOption
// Split a string by delimiter and remove external white space
[[nodiscard]] std::vector<std::string> splitAndTrim(const std::string& s, char delim) const;
[[nodiscard]] std::vector<std::string> splitAndTrim(
const std::string& s,
char delim) const;
// Lowercase for case-insensitive comparisons
std::string toLower(std::string& s) const;
// Does string end with suffix.
// New code should prefer std::string::ends_with directly (C++20). This
// wrapper is kept only for existing callers; do not add new ones.
[[nodiscard]] bool endsWith(const std::string& s, const std::string& suffix) const;
[[nodiscard]] bool endsWith(const std::string& s, const std::string& suffix)
const;
// Conversions
[[nodiscard]] int64_t toIntRange(const std::string& val, int64_t min, int64_t max) const;
[[nodiscard]] int64_t toIntRange(
const std::string& val,
int64_t min,
int64_t max) const;
[[nodiscard]] int32_t toInt32(const std::string& val) const;
[[nodiscard]] int64_t toInt64(const std::string& val) const;
bool toBool(std::string& val) const;
Expand Down
24 changes: 15 additions & 9 deletions libkineto/include/ActivityProfilerInterface.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,9 @@ class ActivityProfilerInterface {
// Call prepareTrace to enable tracing, then run the region to trace
// at least once (and ideally run the same code that is to be traced) to
// allow tracing structures to be initialized.
virtual void prepareTrace([[maybe_unused]] const std::set<ActivityType>& activityTypes,
[[maybe_unused]] const std::string& configStr = "") {}
virtual void prepareTrace(
[[maybe_unused]] const std::set<ActivityType>& activityTypes,
[[maybe_unused]] const std::string& configStr = "") {}

// Toggle GPU tracing as a trace is running to omit certain parts of a graph
virtual void toggleCollectionDynamic([[maybe_unused]] const bool enable) {}
Expand All @@ -77,7 +78,8 @@ class ActivityProfilerInterface {
// FIXME: Pass activityProfiler interface into clientInterface?
virtual void pushCorrelationId([[maybe_unused]] uint64_t id) {}
virtual void popCorrelationId() {}
virtual void transferCpuTrace([[maybe_unused]] std::unique_ptr<CpuTraceBuffer> traceBuffer) {}
virtual void transferCpuTrace(
[[maybe_unused]] std::unique_ptr<CpuTraceBuffer> traceBuffer) {}

// Correlation ids for user defined spans
virtual void pushUserCorrelationId([[maybe_unused]] uint64_t id) {}
Expand All @@ -89,18 +91,22 @@ class ActivityProfilerInterface {

// Record trace metadata, currently supporting only string key and values,
// values with the same key are overwritten
virtual void addMetadata(const std::string& key, const std::string& value) = 0;
virtual void addMetadata(
const std::string& key,
const std::string& value) = 0;

// Add a child activity profiler, this enables frameworks in the application
// to enable custom framework events.
virtual void addChildActivityProfiler([[maybe_unused]] std::unique_ptr<IActivityProfiler> profiler) {}
virtual void addChildActivityProfiler(
[[maybe_unused]] std::unique_ptr<IActivityProfiler> profiler) {}

// Log Invariant Violation to factories enabled. This helps record
// instances when the profiler behaves unexpectedly.
virtual void logInvariantViolation([[maybe_unused]] const std::string& profile_id,
[[maybe_unused]] const std::string& assertion,
[[maybe_unused]] const std::string& error,
[[maybe_unused]] const std::string& group_profile_id = "") {}
virtual void logInvariantViolation(
[[maybe_unused]] const std::string& profile_id,
[[maybe_unused]] const std::string& assertion,
[[maybe_unused]] const std::string& error,
[[maybe_unused]] const std::string& group_profile_id = "") {}
};

} // namespace libkineto
67 changes: 35 additions & 32 deletions libkineto/include/ActivityType.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,16 @@ enum class ActivityType {

XPU_SCOPE_PROFILER = 26, // XPUPTI Profiler scope for performance metrics

ENUM_COUNT = 27, // This is to add buffer and not used for any profiling logic. Add
ENUM_COUNT =
27, // This is to add buffer and not used for any profiling logic. Add
// your new type before it.
OPTIONAL_ACTIVITY_TYPE_START = GLOW_RUNTIME,
};

// Return an array of all activity types except COUNT
constexpr int activityTypeCount = (int)ActivityType::ENUM_COUNT;
constexpr int defaultActivityTypeCount = (int)ActivityType::OPTIONAL_ACTIVITY_TYPE_START;
constexpr int defaultActivityTypeCount =
(int)ActivityType::OPTIONAL_ACTIVITY_TYPE_START;

// These definitions are not part of the public Kineto API. They are inlined
// here because some build configurations include this header
Expand All @@ -72,36 +74,37 @@ struct _ActivityTypeName {
ActivityType type;
};

inline constexpr std::array<_ActivityTypeName, activityTypeCount + 1> _activityTypeNames{{
{"cpu_op", ActivityType::CPU_OP},
{"user_annotation", ActivityType::USER_ANNOTATION},
{"gpu_user_annotation", ActivityType::GPU_USER_ANNOTATION},
{"gpu_memcpy", ActivityType::GPU_MEMCPY},
{"gpu_memset", ActivityType::GPU_MEMSET},
{"kernel", ActivityType::CONCURRENT_KERNEL},
{"external_correlation", ActivityType::EXTERNAL_CORRELATION},
{"cuda_runtime", ActivityType::CUDA_RUNTIME},
{"cuda_driver", ActivityType::CUDA_DRIVER},
{"cpu_instant_event", ActivityType::CPU_INSTANT_EVENT},
{"python_function", ActivityType::PYTHON_FUNCTION},
{"overhead", ActivityType::OVERHEAD},
{"mtia_runtime", ActivityType::MTIA_RUNTIME},
{"mtia_ccp_events", ActivityType::MTIA_CCP_EVENTS},
{"mtia_insight", ActivityType::MTIA_INSIGHT},
{"cuda_sync", ActivityType::CUDA_SYNC},
{"cuda_event", ActivityType::CUDA_EVENT},
{"mtia_counters", ActivityType::MTIA_COUNTERS},
{"glow_runtime", ActivityType::GLOW_RUNTIME},
{"cuda_profiler_range", ActivityType::CUDA_PROFILER_RANGE},
{"hpu_op", ActivityType::HPU_OP},
{"xpu_runtime", ActivityType::XPU_RUNTIME},
{"xpu_driver", ActivityType::XPU_DRIVER},
{"collective_comm", ActivityType::COLLECTIVE_COMM},
{"privateuse1_runtime", ActivityType::PRIVATEUSE1_RUNTIME},
{"privateuse1_driver", ActivityType::PRIVATEUSE1_DRIVER},
{"xpu_scope_profiler", ActivityType::XPU_SCOPE_PROFILER},
{"ENUM_COUNT", ActivityType::ENUM_COUNT},
}};
inline constexpr std::array<_ActivityTypeName, activityTypeCount + 1>
_activityTypeNames{{
{"cpu_op", ActivityType::CPU_OP},
{"user_annotation", ActivityType::USER_ANNOTATION},
{"gpu_user_annotation", ActivityType::GPU_USER_ANNOTATION},
{"gpu_memcpy", ActivityType::GPU_MEMCPY},
{"gpu_memset", ActivityType::GPU_MEMSET},
{"kernel", ActivityType::CONCURRENT_KERNEL},
{"external_correlation", ActivityType::EXTERNAL_CORRELATION},
{"cuda_runtime", ActivityType::CUDA_RUNTIME},
{"cuda_driver", ActivityType::CUDA_DRIVER},
{"cpu_instant_event", ActivityType::CPU_INSTANT_EVENT},
{"python_function", ActivityType::PYTHON_FUNCTION},
{"overhead", ActivityType::OVERHEAD},
{"mtia_runtime", ActivityType::MTIA_RUNTIME},
{"mtia_ccp_events", ActivityType::MTIA_CCP_EVENTS},
{"mtia_insight", ActivityType::MTIA_INSIGHT},
{"cuda_sync", ActivityType::CUDA_SYNC},
{"cuda_event", ActivityType::CUDA_EVENT},
{"mtia_counters", ActivityType::MTIA_COUNTERS},
{"glow_runtime", ActivityType::GLOW_RUNTIME},
{"cuda_profiler_range", ActivityType::CUDA_PROFILER_RANGE},
{"hpu_op", ActivityType::HPU_OP},
{"xpu_runtime", ActivityType::XPU_RUNTIME},
{"xpu_driver", ActivityType::XPU_DRIVER},
{"collective_comm", ActivityType::COLLECTIVE_COMM},
{"privateuse1_runtime", ActivityType::PRIVATEUSE1_RUNTIME},
{"privateuse1_driver", ActivityType::PRIVATEUSE1_DRIVER},
{"xpu_scope_profiler", ActivityType::XPU_SCOPE_PROFILER},
{"ENUM_COUNT", ActivityType::ENUM_COUNT},
}};

inline const char* toString(ActivityType t) {
return _activityTypeNames[static_cast<int>(t)].name;
Expand Down
38 changes: 26 additions & 12 deletions libkineto/include/Config.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ class Config : public AbstractConfig {
}

[[nodiscard]] bool activityProfilerEnabled() const {
return activityProfilerEnabled_ || activitiesOnDemandTimestamp_.time_since_epoch().count() > 0;
return activityProfilerEnabled_ ||
activitiesOnDemandTimestamp_.time_since_epoch().count() > 0;
}

// Log activitiy trace to this file
Expand Down Expand Up @@ -171,7 +172,8 @@ class Config : public AbstractConfig {
// Monitor profiling threads and report when a thread is not responding
// for a given number of seconds.
// A period of 0 means disable.
[[nodiscard]] std::chrono::seconds eventProfilerHeartbeatMonitorPeriod() const {
[[nodiscard]] std::chrono::seconds eventProfilerHeartbeatMonitorPeriod()
const {
return eventProfilerHeartbeatMonitorPeriod_;
}

Expand Down Expand Up @@ -240,7 +242,8 @@ class Config : public AbstractConfig {
activitiesCudaSyncWaitEvents_ = enable;
}

[[nodiscard]] std::chrono::time_point<std::chrono::system_clock> requestTimestamp() const {
[[nodiscard]] std::chrono::time_point<std::chrono::system_clock>
requestTimestamp() const {
return profileStartTime_;
}

Expand Down Expand Up @@ -295,20 +298,25 @@ class Config : public AbstractConfig {
return onDemandConfigUpdateIntervalSecs_;
}

static std::chrono::milliseconds alignUp(std::chrono::milliseconds duration, std::chrono::milliseconds alignment) {
static std::chrono::milliseconds alignUp(
std::chrono::milliseconds duration,
std::chrono::milliseconds alignment) {
duration += alignment;
return duration - (duration % alignment);
}

[[nodiscard]] std::chrono::time_point<std::chrono::system_clock> eventProfilerOnDemandStartTime() const {
[[nodiscard]] std::chrono::time_point<std::chrono::system_clock>
eventProfilerOnDemandStartTime() const {
return eventProfilerOnDemandTimestamp_;
}

[[nodiscard]] std::chrono::time_point<std::chrono::system_clock> eventProfilerOnDemandEndTime() const {
[[nodiscard]] std::chrono::time_point<std::chrono::system_clock>
eventProfilerOnDemandEndTime() const {
return eventProfilerOnDemandTimestamp_ + eventProfilerOnDemandDuration_;
}

[[nodiscard]] std::chrono::time_point<std::chrono::system_clock> activityProfilerRequestReceivedTime() const {
[[nodiscard]] std::chrono::time_point<std::chrono::system_clock>
activityProfilerRequestReceivedTime() const {
return activitiesOnDemandTimestamp_;
}

Expand Down Expand Up @@ -351,9 +359,12 @@ class Config : public AbstractConfig {
void printActivityProfilerConfig(std::ostream& s) const override;
void setActivityDependentConfig() override;

void validate(const std::chrono::time_point<std::chrono::system_clock>& fallbackProfileStartTime) override;
void validate(const std::chrono::time_point<std::chrono::system_clock>&
fallbackProfileStartTime) override;

static void addConfigFactory(std::string name, std::function<AbstractConfig*(Config&)> factory);
static void addConfigFactory(
std::string name,
std::function<AbstractConfig*(Config&)> factory);

void print(std::ostream& s) const;

Expand Down Expand Up @@ -382,7 +393,8 @@ class Config : public AbstractConfig {
private:
explicit Config(const Config& other) = default;

AbstractConfig* cloneDerived([[maybe_unused]] AbstractConfig& parent) const override {
AbstractConfig* cloneDerived(
[[maybe_unused]] AbstractConfig& parent) const override {
// Clone from AbstractConfig not supported
assert(false);
return nullptr;
Expand Down Expand Up @@ -416,7 +428,8 @@ class Config : public AbstractConfig {
// On-demand duration
std::chrono::seconds eventProfilerOnDemandDuration_;
// Last on-demand request
std::chrono::time_point<std::chrono::system_clock> eventProfilerOnDemandTimestamp_;
std::chrono::time_point<std::chrono::system_clock>
eventProfilerOnDemandTimestamp_;

int eventProfilerMaxInstancesPerGpu_;

Expand Down Expand Up @@ -476,7 +489,8 @@ class Config : public AbstractConfig {
// Only profile nets with at least this many GPU operators
int activitiesExternalAPIGpuOpCountThreshold_;
// Last activity profiler request
std::chrono::time_point<std::chrono::system_clock> activitiesOnDemandTimestamp_;
std::chrono::time_point<std::chrono::system_clock>
activitiesOnDemandTimestamp_;

// ActivityProfilers are triggered by either:
// Synchronized start timestamps
Expand Down
5 changes: 3 additions & 2 deletions libkineto/include/EnvMetadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@ inline std::unordered_map<std::string, std::string> getEnvMetadata() {
}

// Capture hostname for per-rank host identification in distributed training.
// $HOSTNAME is not guaranteed in non-interactive or containerized environments,
// so we use gethostname() which reads the kernel hostname directly.
// $HOSTNAME is not guaranteed in non-interactive or containerized
// environments, so we use gethostname() which reads the kernel hostname
// directly.
std::array<char, 256> hostname{};
if (gethostname(hostname.data(), hostname.size()) == 0) {
hostname.back() = '\0';
Expand Down
26 changes: 19 additions & 7 deletions libkineto/include/GenericTraceActivity.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@ constexpr unsigned int kLinkAsyncCpuGpu = 2;
// @lint-ignore-every CLANGTIDY cppcoreguidelines-pro-type-member-init
class GenericTraceActivity : public ITraceActivity {
public:
GenericTraceActivity() : activityType(ActivityType::ENUM_COUNT), traceSpan_(nullptr) {}
GenericTraceActivity()
: activityType(ActivityType::ENUM_COUNT), traceSpan_(nullptr) {}

GenericTraceActivity(const TraceSpan& trace, ActivityType type, const std::string& name)
GenericTraceActivity(
const TraceSpan& trace,
ActivityType type,
const std::string& name)
: activityType(type), activityName(name), traceSpan_(&trace) {}

int64_t deviceId() const override {
Expand Down Expand Up @@ -107,11 +111,14 @@ class GenericTraceActivity : public ITraceActivity {
// Typed metadata: the value is stored as the field's declared type
template <typename T, typename V>
void addMetadata(const MetadataField<T>& field, const V& value) {
static_assert(std::is_same_v<T, std::decay_t<V>>, "value type must match field's declared type");
static_assert(
std::is_same_v<T, std::decay_t<V>>,
"value type must match field's declared type");
metadataMap_.emplace(std::string{field.name}, TypedValue{value});
}

// Adds typed metadata dynamically by key. Catalog registration is not required.
// Adds typed metadata dynamically by key. Catalog registration is not
// required.
void addTypedMetadata(std::string_view key, TypedValue value) {
metadataMap_.emplace(std::string{key}, std::move(value));
}
Expand All @@ -128,7 +135,8 @@ class GenericTraceActivity : public ITraceActivity {
counterValues_.emplace_back(name, value);
}

const std::vector<std::pair<std::string, double>>& counterValues() const override {
const std::vector<std::pair<std::string, double>>& counterValues()
const override {
return counterValues_;
}

Expand All @@ -152,8 +160,12 @@ class GenericTraceActivity : public ITraceActivity {
void visitTypedMetadata(ITypedMetadataVisitor& visitor) const override {
// Dynamically build a MetadataField during visit
for (const auto& kv : metadataMap_) {
std::visit([&](const auto& v) { visitor.visit(MetadataField<std::decay_t<decltype(v)>>{kv.first}, v); },
kv.second);
std::visit(
[&](const auto& v) {
visitor.visit(
MetadataField<std::decay_t<decltype(v)>>{kv.first}, v);
},
kv.second);
}
}

Expand Down
Loading
Loading