-
Notifications
You must be signed in to change notification settings - Fork 564
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement go.schedule.duration (#5991)
Fixes #5974. This uses the same method for estimating the sum as the prometheus client-go: https://github.com/prometheus/client_golang/blob/46f77a97fa1e83e7285634745bd5c92c11bf20c7/prometheus/go_collector_latest.go#L543, which is to use the lower-bound of each bucket as the value of each observation. It implements this using a metric producer. It is designed to be used together with existing runtime instrumentation. --------- Co-authored-by: Sam Xie <sam@samxie.me>
- Loading branch information
Showing
17 changed files
with
423 additions
and
365 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package runtime_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "log" | ||
| "time" | ||
|
|
||
| "go.opentelemetry.io/contrib/instrumentation/runtime" | ||
| "go.opentelemetry.io/otel" | ||
| "go.opentelemetry.io/otel/sdk/metric" | ||
| ) | ||
|
|
||
| func Example() { | ||
| // This reader is used as a stand-in for a reader that will actually export | ||
| // data. See https://pkg.go.dev/go.opentelemetry.io/otel/exporters for | ||
| // exporters that can be used as or with readers. | ||
| reader := metric.NewManualReader( | ||
| // Add the runtime producer to get histograms from the Go runtime. | ||
| metric.WithProducer(runtime.NewProducer()), | ||
| ) | ||
| provider := metric.NewMeterProvider(metric.WithReader(reader)) | ||
| defer func() { | ||
| err := provider.Shutdown(context.Background()) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| }() | ||
| otel.SetMeterProvider(provider) | ||
|
|
||
| // Start go runtime metric collection. | ||
| err := runtime.Start(runtime.WithMinimumReadMemStatsInterval(time.Second)) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package runtime // import "go.opentelemetry.io/contrib/instrumentation/runtime" | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "math" | ||
| "runtime/metrics" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "go.opentelemetry.io/otel/attribute" | ||
| "go.opentelemetry.io/otel/sdk/instrumentation" | ||
| "go.opentelemetry.io/otel/sdk/metric" | ||
| "go.opentelemetry.io/otel/sdk/metric/metricdata" | ||
| ) | ||
|
|
||
| var startTime time.Time | ||
|
|
||
| func init() { | ||
| startTime = time.Now() | ||
| } | ||
|
|
||
| var histogramMetrics = []string{goSchedLatencies} | ||
|
|
||
| // Producer is a metric.Producer, which provides precomputed histogram metrics from the go runtime. | ||
| type Producer struct { | ||
| lock sync.Mutex | ||
| collector *goCollector | ||
| } | ||
|
|
||
| var _ metric.Producer = (*Producer)(nil) | ||
|
|
||
| // NewProducer creates a Producer which provides precomputed histogram metrics from the go runtime. | ||
| func NewProducer(opts ...ProducerOption) *Producer { | ||
| c := newProducerConfig(opts...) | ||
| return &Producer{ | ||
| collector: newCollector(c.MinimumReadMemStatsInterval, histogramMetrics), | ||
| } | ||
| } | ||
|
|
||
| // Produce returns precomputed histogram metrics from the go runtime, or an error if unsuccessful. | ||
| func (p *Producer) Produce(context.Context) ([]metricdata.ScopeMetrics, error) { | ||
| p.lock.Lock() | ||
| p.collector.refresh() | ||
| schedHist := p.collector.getHistogram(goSchedLatencies) | ||
| p.lock.Unlock() | ||
| // Use the last collection time (which may or may not be now) for the timestamp. | ||
| histDp := convertRuntimeHistogram(schedHist, p.collector.lastCollect) | ||
| if len(histDp) == 0 { | ||
| return nil, fmt.Errorf("unable to obtain go.schedule.duration metric from the runtime") | ||
| } | ||
| return []metricdata.ScopeMetrics{ | ||
| { | ||
| Scope: instrumentation.Scope{ | ||
| Name: ScopeName, | ||
| Version: Version(), | ||
| }, | ||
| Metrics: []metricdata.Metrics{ | ||
| { | ||
| Name: "go.schedule.duration", | ||
| Description: "The time goroutines have spent in the scheduler in a runnable state before actually running.", | ||
| Unit: "s", | ||
| Data: metricdata.Histogram[float64]{ | ||
| Temporality: metricdata.CumulativeTemporality, | ||
| DataPoints: histDp, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, nil | ||
| } | ||
|
|
||
| var emptySet = attribute.EmptySet() | ||
|
|
||
| func convertRuntimeHistogram(runtimeHist *metrics.Float64Histogram, ts time.Time) []metricdata.HistogramDataPoint[float64] { | ||
| if runtimeHist == nil { | ||
| return nil | ||
| } | ||
| bounds := runtimeHist.Buckets | ||
| counts := runtimeHist.Counts | ||
| if len(bounds) < 2 { | ||
| // runtime histograms are guaranteed to have at least two bucket boundaries. | ||
| return nil | ||
| } | ||
| // trim the first bucket since it is a lower bound. OTel histogram boundaries only have an upper bound. | ||
| bounds = bounds[1:] | ||
| if bounds[len(bounds)-1] == math.Inf(1) { | ||
| // trim the last bucket if it is +Inf, since the +Inf boundary is implicit in OTel. | ||
| bounds = bounds[:len(bounds)-1] | ||
| } else { | ||
| // if the last bucket is not +Inf, append an extra zero count since | ||
| // the implicit +Inf bucket won't have any observations. | ||
| counts = append(counts, 0) | ||
| } | ||
| count := uint64(0) | ||
| sum := float64(0) | ||
| for i, c := range counts { | ||
| count += c | ||
| // This computed sum is an underestimate, since it assumes each | ||
| // observation happens at the bucket's lower bound. | ||
| if i > 0 && count != 0 { | ||
| sum += bounds[i-1] * float64(count) | ||
| } | ||
| } | ||
|
|
||
| return []metricdata.HistogramDataPoint[float64]{ | ||
| { | ||
| StartTime: startTime, | ||
| Count: count, | ||
| Sum: sum, | ||
| Time: ts, | ||
| Bounds: bounds, | ||
| BucketCounts: counts, | ||
| Attributes: *emptySet, | ||
| }, | ||
| } | ||
| } |
Oops, something went wrong.