Skip to content

Commit

Permalink
[Metric] Standardise histogram metric output for prometheus (apache#5671
Browse files Browse the repository at this point in the history 
)

Update histogram metric's output to prometheus standard, the output
like following:

test_registry_task_duration{quantile="0.50"} 50
test_registry_task_duration{quantile="0.75"} 75
test_registry_task_duration{quantile="0.90"} 95.8333
test_registry_task_duration{quantile="0.95"} 100
test_registry_task_duration{quantile="0.99"} 100
test_registry_task_duration_sum 5050
test_registry_task_duration_count 100
  • Loading branch information
@acelyc111
acelyc111 authored Apr 20, 2021
1 parent 2b7d7e3 commit caa7af3
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 34 deletions.
97 changes: 68 additions & 29 deletions be/src/util/metrics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,31 +77,49 @@ const char* unit_name(MetricUnit unit) {
}
}

std::string labels_to_string(const Labels& entity_labels, const Labels& metric_labels) {
if (entity_labels.empty() && metric_labels.empty()) {
std::string labels_to_string(std::initializer_list<const Labels*> multi_labels) {
bool all_empty = true;
for (const auto& labels : multi_labels) {
if (!labels->empty()) {
all_empty = false;
break;
}
}
if (all_empty) {
return std::string();
}

std::stringstream ss;
ss << "{";
int i = 0;
for (const auto& label : entity_labels) {
if (i++ > 0) {
ss << ",";
}
ss << label.first << "=\"" << label.second << "\"";
}
for (const auto& label : metric_labels) {
if (i++ > 0) {
ss << ",";
for (auto labels : multi_labels) {
for (const auto& label : *labels) {
if (i++ > 0) {
ss << ",";
}
ss << label.first << "=\"" << label.second << "\"";
}
ss << label.first << "=\"" << label.second << "\"";
}
ss << "}";

return ss.str();
}

std::string Metric::to_prometheus(const std::string& display_name,
const Labels& entity_labels,
const Labels& metric_labels) const {
std::stringstream ss;
ss << display_name // metric name
<< labels_to_string({&entity_labels, &metric_labels}) // metric labels
<< " " << to_string() << "\n"; // metric value
return ss.str();
}

std::map<std::string, double> HistogramMetric::_s_output_percentiles = {{"0.50", 50.0},
{"0.75", 75.0},
{"0.90", 90.0},
{"0.95", 95.0},
{"0.99", 99.0}};
void HistogramMetric::clear() {
std::lock_guard<SpinLock> l(_lock);
_stats.clear();
Expand Down Expand Up @@ -140,20 +158,36 @@ std::string HistogramMetric::to_string() const {
return _stats.to_string();
}

rj::Value HistogramMetric::to_json_value() const {
rj::Document document;
rj::Document::AllocatorType& allocator = document.GetAllocator();
rj::Value json_value(rj::kObjectType);
std::string HistogramMetric::to_prometheus(const std::string& display_name,
const Labels& entity_labels,
const Labels& metric_labels) const {
std::stringstream ss;
for (const auto& percentile : _s_output_percentiles) {
auto quantile_lable = Labels({{"quantile", percentile.first}});
ss << display_name
<< labels_to_string({&entity_labels, &metric_labels, &quantile_lable})
<< " " << _stats.percentile(percentile.second) << "\n";
}
ss << display_name << "_sum"
<< labels_to_string({&entity_labels, &metric_labels})
<< " " << _stats.sum() << "\n";
ss << display_name << "_count"
<< labels_to_string({&entity_labels, &metric_labels})
<< " " << _stats.num() << "\n";

return ss.str();
}

rj::Value HistogramMetric::to_json_value(rj::Document::AllocatorType& allocator) const {
rj::Value json_value(rj::kObjectType);
json_value.AddMember("total_count", rj::Value(_stats.num()), allocator);
json_value.AddMember("min", rj::Value(_stats.min()), allocator);
json_value.AddMember("average", rj::Value(_stats.average()), allocator);
json_value.AddMember("median", rj::Value(_stats.median()), allocator);
json_value.AddMember("percentile_75", rj::Value(_stats.percentile(75.0)), allocator);
json_value.AddMember("percentile_95", rj::Value(_stats.percentile(95)), allocator);
json_value.AddMember("percentile_99", rj::Value(_stats.percentile(99)), allocator);
json_value.AddMember("percentile_99_9", rj::Value(_stats.percentile(99.9)), allocator);
json_value.AddMember("percentile_99_99", rj::Value(_stats.percentile(99.99)), allocator);
for (const auto& percentile : _s_output_percentiles) {
json_value.AddMember(rj::Value(std::string("percentile_").append(percentile.first.substr(2)).c_str(), allocator),
rj::Value(_stats.percentile(percentile.second)), allocator);
}
json_value.AddMember("standard_deviation", rj::Value(_stats.standard_deviation()), allocator);
json_value.AddMember("max", rj::Value(_stats.max()), allocator);
json_value.AddMember("total_sum", rj::Value(_stats.sum()), allocator);
Expand All @@ -169,6 +203,12 @@ std::string MetricPrototype::combine_name(const std::string& registry_name) cons
return (registry_name.empty() ? std::string() : registry_name + "_") + simple_name();
}

std::string MetricPrototype::to_prometheus(const std::string& registry_name) const {
std::stringstream ss;
ss << "# TYPE " << combine_name(registry_name) << " " << type << "\n";
return ss.str();
}

void MetricEntity::deregister_metric(const MetricPrototype* metric_type) {
std::lock_guard<SpinLock> l(_lock);
auto metric = _metrics.find(metric_type);
Expand Down Expand Up @@ -260,7 +300,6 @@ void MetricRegistry::trigger_all_hooks(bool force) const {
}

std::string MetricRegistry::to_prometheus(bool with_tablet_metrics) const {
std::stringstream ss;
// Reorder by MetricPrototype
EntityMetricsByType entity_metrics_by_types;
std::lock_guard<SpinLock> l(_lock);
Expand All @@ -282,21 +321,21 @@ std::string MetricRegistry::to_prometheus(bool with_tablet_metrics) const {
}
}
}

// Output
std::stringstream ss;
std::string last_group_name;
for (const auto& entity_metrics_by_type : entity_metrics_by_types) {
if (last_group_name.empty() ||
last_group_name != entity_metrics_by_type.first->group_name) {
ss << "# TYPE " << entity_metrics_by_type.first->combine_name(_name) << " "
<< entity_metrics_by_type.first->type << "\n"; // metric TYPE line
ss << entity_metrics_by_type.first->to_prometheus(_name); // metric TYPE line
}
last_group_name = entity_metrics_by_type.first->group_name;
std::string display_name = entity_metrics_by_type.first->combine_name(_name);
for (const auto& entity_metric : entity_metrics_by_type.second) {
ss << display_name // metric name
<< labels_to_string(entity_metric.first->_labels,
entity_metrics_by_type.first->labels) // metric labels
<< " " << entity_metric.second->to_string() << "\n"; // metric value
ss << entity_metric.second->to_prometheus(display_name, // metric key-value line
entity_metric.first->_labels,
entity_metrics_by_type.first->labels);
}
}

Expand Down Expand Up @@ -334,7 +373,7 @@ std::string MetricRegistry::to_json(bool with_tablet_metrics) const {
rj::Value unit_val(unit_name(metric.first->unit), allocator);
metric_obj.AddMember("unit", unit_val, allocator);
// value
metric_obj.AddMember("value", metric.second->to_json_value(), allocator);
metric_obj.AddMember("value", metric.second->to_json_value(allocator), allocator);
doc.PushBack(metric_obj, allocator);
}
}
Expand Down
20 changes: 15 additions & 5 deletions be/src/util/metrics.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,17 @@ enum class MetricUnit {
std::ostream& operator<<(std::ostream& os, MetricType type);
const char* unit_name(MetricUnit unit);

using Labels = std::unordered_map<std::string, std::string>;

class Metric {
public:
Metric() {}
virtual ~Metric() {}
virtual std::string to_string() const = 0;
virtual rj::Value to_json_value() const = 0;
virtual std::string to_prometheus(const std::string& display_name,
const Labels& entity_labels,
const Labels& metric_labels) const;
virtual rj::Value to_json_value(rj::Document::AllocatorType& allocator) const = 0;

private:
friend class MetricRegistry;
Expand All @@ -89,7 +94,7 @@ class AtomicMetric : public Metric {

void set_value(const T& value) { _value.store(value); }

rj::Value to_json_value() const override { return rj::Value(value()); }
rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { return rj::Value(value()); }

protected:
std::atomic<T> _value;
Expand Down Expand Up @@ -118,7 +123,7 @@ class LockSimpleMetric : public Metric {
_value = value;
}

rj::Value to_json_value() const override { return rj::Value(value()); }
rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { return rj::Value(value()); }

protected:
// We use spinlock instead of std::atomic is because atomic don't support
Expand Down Expand Up @@ -154,7 +159,7 @@ class CoreLocalCounter : public Metric {

void increment(const T& delta) { __sync_fetch_and_add(_value.access(), delta); }

rj::Value to_json_value() const override { return rj::Value(value()); }
rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { return rj::Value(value()); }

protected:
CoreLocalValue<T> _value;
Expand Down Expand Up @@ -182,9 +187,13 @@ class HistogramMetric : public Metric {
double average() const;
double standard_deviation() const;
std::string to_string() const override;
rj::Value to_json_value() const override;
std::string to_prometheus(const std::string& display_name,
const Labels& entity_labels,
const Labels& metric_labels) const override;
rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override;

protected:
static std::map<std::string, double> _s_output_percentiles;
mutable SpinLock _lock;
HistogramStat _stats;
};
Expand Down Expand Up @@ -242,6 +251,7 @@ struct MetricPrototype {

std::string simple_name() const;
std::string combine_name(const std::string& registry_name) const;
std::string to_prometheus(const std::string& registry_name) const;

bool is_core_metric;
MetricType type;
Expand Down
65 changes: 65 additions & 0 deletions be/test/util/new_metrics_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,71 @@ test_registry_cpu{mode="guest"} 58
registry.deregister_entity(entity);
}
}

TEST_F(MetricsTest, HistogramRegistryOutput) {
MetricRegistry registry("test_registry");

{
// Register one histogram metric to the entity
auto entity = registry.register_entity("test_entity");

MetricPrototype task_duration_type(MetricType::HISTOGRAM,
MetricUnit::MILLISECONDS,
"task_duration");
HistogramMetric* task_duration = (HistogramMetric*)entity->register_metric<HistogramMetric>(&task_duration_type);
for (int j = 1; j <= 100; j++) {
task_duration->add(j);
}

ASSERT_EQ(R"(# TYPE test_registry_task_duration histogram
test_registry_task_duration{quantile="0.50"} 50
test_registry_task_duration{quantile="0.75"} 75
test_registry_task_duration{quantile="0.90"} 95.8333
test_registry_task_duration{quantile="0.95"} 100
test_registry_task_duration{quantile="0.99"} 100
test_registry_task_duration_sum 5050
test_registry_task_duration_count 100
)",
registry.to_prometheus());
ASSERT_EQ(R"*([{"tags":{"metric":"task_duration"},"unit":"milliseconds",)*"
R"*("value":{"total_count":100,"min":1,"average":50.5,"median":50.0,)*"
R"*("percentile_50":50.0,"percentile_75":75.0,"percentile_90":95.83333333333334,"percentile_95":100.0,"percentile_99":100.0,)*"
R"*("standard_deviation":28.86607004772212,"max":100,"total_sum":5050}}])*",
registry.to_json());
registry.deregister_entity(entity);
}

{
// Register one histogram metric with lables to the entity
auto entity = registry.register_entity("test_entity", {{"instance", "test"}});

MetricPrototype task_duration_type(MetricType::HISTOGRAM,
MetricUnit::MILLISECONDS,
"task_duration", "", "",
{{"type", "create_tablet"}});
HistogramMetric* task_duration = (HistogramMetric*)entity->register_metric<HistogramMetric>(&task_duration_type);
for (int j = 1; j <= 100; j++) {
task_duration->add(j);
}

ASSERT_EQ(R"(# TYPE test_registry_task_duration histogram
test_registry_task_duration{instance="test",type="create_tablet",quantile="0.50"} 50
test_registry_task_duration{instance="test",type="create_tablet",quantile="0.75"} 75
test_registry_task_duration{instance="test",type="create_tablet",quantile="0.90"} 95.8333
test_registry_task_duration{instance="test",type="create_tablet",quantile="0.95"} 100
test_registry_task_duration{instance="test",type="create_tablet",quantile="0.99"} 100
test_registry_task_duration_sum{instance="test",type="create_tablet"} 5050
test_registry_task_duration_count{instance="test",type="create_tablet"} 100
)",
registry.to_prometheus());
ASSERT_EQ(R"*([{"tags":{"metric":"task_duration","type":"create_tablet","instance":"test"},"unit":"milliseconds",)*"
R"*("value":{"total_count":100,"min":1,"average":50.5,"median":50.0,)*"
R"*("percentile_50":50.0,"percentile_75":75.0,"percentile_90":95.83333333333334,"percentile_95":100.0,"percentile_99":100.0,)*"
R"*("standard_deviation":28.86607004772212,"max":100,"total_sum":5050}}])*",
registry.to_json());
registry.deregister_entity(entity);
}
}
} // namespace doris

int main(int argc, char** argv) {
Expand Down

0 comments on commit caa7af3

Please sign in to comment.