From 0d4b7c77a1a66417acb01c9c0caefa3067dec34d Mon Sep 17 00:00:00 2001 From: Wang Guan Date: Thu, 20 Jul 2023 10:24:48 +0800 Subject: [PATCH] feat: add requests total and request succeed total aggregate metrics --- metrics/prometheus/metric_set.go | 32 ++++++++++++++++++------------ metrics/prometheus/model.go | 34 ++++++++++++++++++++++++++++---- metrics/prometheus/reporter.go | 4 ++++ 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/metrics/prometheus/metric_set.go b/metrics/prometheus/metric_set.go index 488fce1319..5d3caf2736 100644 --- a/metrics/prometheus/metric_set.go +++ b/metrics/prometheus/metric_set.go @@ -43,17 +43,19 @@ func (ms *metricSet) init(reporterConfig *metrics.ReporterConfig) { } type rpcCommonMetrics struct { - qpsTotal *qpsGaugeVec - requestsTotal *prometheus.CounterVec - requestsProcessingTotal *prometheus.GaugeVec - requestsSucceedTotal *prometheus.CounterVec - rtMillisecondsMin *GaugeVecWithSyncMap - rtMillisecondsMax *GaugeVecWithSyncMap - rtMillisecondsSum *prometheus.CounterVec - rtMillisecondsAvg *GaugeVecWithSyncMap - rtMillisecondsLast *prometheus.GaugeVec - rtMillisecondsQuantiles *quantileGaugeVec - rtMillisecondsAggregate *aggregatorGaugeVec + qpsTotal *qpsGaugeVec + requestsTotal *prometheus.CounterVec + requestsTotalAggregate *aggregateCounterGaugeVec + requestsProcessingTotal *prometheus.GaugeVec + requestsSucceedTotal *prometheus.CounterVec + requestsSucceedTotalAggregate *aggregateCounterGaugeVec + rtMillisecondsMin *GaugeVecWithSyncMap + rtMillisecondsMax *GaugeVecWithSyncMap + rtMillisecondsSum *prometheus.CounterVec + rtMillisecondsAvg *GaugeVecWithSyncMap + rtMillisecondsLast *prometheus.GaugeVec + rtMillisecondsQuantiles *quantileGaugeVec + rtMillisecondsAggregate *aggregateFunctionsGaugeVec } type providerMetrics struct { @@ -63,15 +65,17 @@ type providerMetrics struct { func (pm *providerMetrics) init(reporterConfig *metrics.ReporterConfig) { pm.qpsTotal = newQpsGaugeVec(buildMetricsName(providerField, qpsField, totalField), reporterConfig.Namespace, labelNames) pm.requestsTotal = newAutoCounterVec(buildMetricsName(providerField, requestsField, totalField), reporterConfig.Namespace, labelNames) + pm.requestsTotalAggregate = newAggregateCounterGaugeVec(buildMetricsName(providerField, requestsField, totalField, aggregateField), reporterConfig.Namespace, labelNames) pm.requestsProcessingTotal = newAutoGaugeVec(buildMetricsName(providerField, requestsField, processingField, totalField), reporterConfig.Namespace, labelNames) pm.requestsSucceedTotal = newAutoCounterVec(buildMetricsName(providerField, requestsField, succeedField, totalField), reporterConfig.Namespace, labelNames) + pm.requestsSucceedTotalAggregate = newAggregateCounterGaugeVec(buildMetricsName(providerField, requestsField, succeedField, totalField, aggregateField), reporterConfig.Namespace, labelNames) pm.rtMillisecondsMin = newAutoGaugeVecWithSyncMap(buildMetricsName(providerField, rtField, milliSecondsField, minField), reporterConfig.Namespace, labelNames) pm.rtMillisecondsMax = newAutoGaugeVecWithSyncMap(buildMetricsName(providerField, rtField, milliSecondsField, maxField), reporterConfig.Namespace, labelNames) pm.rtMillisecondsSum = newAutoCounterVec(buildMetricsName(providerField, rtField, milliSecondsField, sumField), reporterConfig.Namespace, labelNames) pm.rtMillisecondsAvg = newAutoGaugeVecWithSyncMap(buildMetricsName(providerField, rtField, milliSecondsField, avgField), reporterConfig.Namespace, labelNames) pm.rtMillisecondsLast = newAutoGaugeVec(buildMetricsName(providerField, rtField, milliSecondsField, lastField), reporterConfig.Namespace, labelNames) pm.rtMillisecondsQuantiles = newQuantileGaugeVec(buildRTQuantilesMetricsNames(providerField, quantiles), reporterConfig.Namespace, labelNames, quantiles) - pm.rtMillisecondsAggregate = newAggregatorGaugeVec( + pm.rtMillisecondsAggregate = newAggregateFunctionsGaugeVec( buildMetricsName(providerField, rtField, minField, milliSecondsField, aggregateField), buildMetricsName(providerField, rtField, maxField, milliSecondsField, aggregateField), buildMetricsName(providerField, rtField, avgField, milliSecondsField, aggregateField), @@ -87,15 +91,17 @@ type consumerMetrics struct { func (cm *consumerMetrics) init(reporterConfig *metrics.ReporterConfig) { cm.qpsTotal = newQpsGaugeVec(buildMetricsName(consumerField, qpsField, totalField), reporterConfig.Namespace, labelNames) cm.requestsTotal = newAutoCounterVec(buildMetricsName(consumerField, requestsField, totalField), reporterConfig.Namespace, labelNames) + cm.requestsTotalAggregate = newAggregateCounterGaugeVec(buildMetricsName(consumerField, requestsField, totalField, aggregateField), reporterConfig.Namespace, labelNames) cm.requestsProcessingTotal = newAutoGaugeVec(buildMetricsName(consumerField, requestsField, processingField, totalField), reporterConfig.Namespace, labelNames) cm.requestsSucceedTotal = newAutoCounterVec(buildMetricsName(consumerField, requestsField, succeedField, totalField), reporterConfig.Namespace, labelNames) + cm.requestsSucceedTotalAggregate = newAggregateCounterGaugeVec(buildMetricsName(consumerField, requestsField, succeedField, totalField, aggregateField), reporterConfig.Namespace, labelNames) cm.rtMillisecondsMin = newAutoGaugeVecWithSyncMap(buildMetricsName(consumerField, rtField, milliSecondsField, minField), reporterConfig.Namespace, labelNames) cm.rtMillisecondsMax = newAutoGaugeVecWithSyncMap(buildMetricsName(consumerField, rtField, milliSecondsField, maxField), reporterConfig.Namespace, labelNames) cm.rtMillisecondsSum = newAutoCounterVec(buildMetricsName(consumerField, rtField, milliSecondsField, sumField), reporterConfig.Namespace, labelNames) cm.rtMillisecondsAvg = newAutoGaugeVecWithSyncMap(buildMetricsName(consumerField, rtField, milliSecondsField, avgField), reporterConfig.Namespace, labelNames) cm.rtMillisecondsLast = newAutoGaugeVec(buildMetricsName(consumerField, rtField, milliSecondsField, lastField), reporterConfig.Namespace, labelNames) cm.rtMillisecondsQuantiles = newQuantileGaugeVec(buildRTQuantilesMetricsNames(consumerField, quantiles), reporterConfig.Namespace, labelNames, quantiles) - cm.rtMillisecondsAggregate = newAggregatorGaugeVec( + cm.rtMillisecondsAggregate = newAggregateFunctionsGaugeVec( buildMetricsName(consumerField, rtField, minField, milliSecondsField, aggregateField), buildMetricsName(consumerField, rtField, maxField, milliSecondsField, aggregateField), buildMetricsName(consumerField, rtField, avgField, milliSecondsField, aggregateField), diff --git a/metrics/prometheus/model.go b/metrics/prometheus/model.go index bda08a8315..fbfeb7a94f 100644 --- a/metrics/prometheus/model.go +++ b/metrics/prometheus/model.go @@ -325,15 +325,41 @@ func (gv *qpsGaugeVec) updateQps(labels *prometheus.Labels) { } } -type aggregatorGaugeVec struct { +type aggregateCounterGaugeVec struct { + gaugeVec *prometheus.GaugeVec + syncMap *sync.Map // key: labels string, value: TimeWindowCounter +} + +func newAggregateCounterGaugeVec(name, namespace string, labels []string) *aggregateCounterGaugeVec { + return &aggregateCounterGaugeVec{ + gaugeVec: newAutoGaugeVec(name, namespace, labels), + syncMap: &sync.Map{}, + } +} + +func (gv *aggregateCounterGaugeVec) inc(labels *prometheus.Labels) { + key := convertLabelsToMapKey(*labels) + cur := aggregate.NewTimeWindowCounter(10, 120) + cur.Inc() + + if actual, loaded := gv.syncMap.LoadOrStore(key, cur); loaded { + store := actual.(*aggregate.TimeWindowCounter) + store.Inc() + gv.gaugeVec.With(*labels).Set(store.Count()) + } else { + gv.gaugeVec.With(*labels).Set(cur.Count()) + } +} + +type aggregateFunctionsGaugeVec struct { min *prometheus.GaugeVec max *prometheus.GaugeVec avg *prometheus.GaugeVec syncMap *sync.Map // key: labels string, value: TimeWindowAggregator } -func newAggregatorGaugeVec(minName, maxName, avgName, namespace string, labels []string) *aggregatorGaugeVec { - return &aggregatorGaugeVec{ +func newAggregateFunctionsGaugeVec(minName, maxName, avgName, namespace string, labels []string) *aggregateFunctionsGaugeVec { + return &aggregateFunctionsGaugeVec{ min: newAutoGaugeVec(minName, namespace, labels), max: newAutoGaugeVec(maxName, namespace, labels), avg: newAutoGaugeVec(avgName, namespace, labels), @@ -341,7 +367,7 @@ func newAggregatorGaugeVec(minName, maxName, avgName, namespace string, labels [ } } -func (gv *aggregatorGaugeVec) update(labels *prometheus.Labels, curValue int64) { +func (gv *aggregateFunctionsGaugeVec) update(labels *prometheus.Labels, curValue int64) { key := convertLabelsToMapKey(*labels) cur := aggregate.NewTimeWindowAggregator(10, 120) cur.Add(float64(curValue)) diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index db458504b7..715fd97ca8 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -155,8 +155,10 @@ func (reporter *PrometheusReporter) incRequestsTotal(role string, labels *promet switch role { case providerField: reporter.provider.requestsTotal.With(*labels).Inc() + reporter.provider.requestsTotalAggregate.inc(labels) case consumerField: reporter.consumer.requestsTotal.With(*labels).Inc() + reporter.consumer.requestsTotalAggregate.inc(labels) } } @@ -182,8 +184,10 @@ func (reporter *PrometheusReporter) incRequestsSucceedTotal(role string, labels switch role { case providerField: reporter.provider.requestsSucceedTotal.With(*labels).Inc() + reporter.provider.requestsSucceedTotalAggregate.inc(labels) case consumerField: reporter.consumer.requestsSucceedTotal.With(*labels).Inc() + reporter.consumer.requestsSucceedTotalAggregate.inc(labels) } }