Skip to content

Commit

Permalink
otlptracehttp, otlpmetrichttp: Retry for 502, 504 HTTP statuses (#4670)
Browse files Browse the repository at this point in the history
  • Loading branch information
pellared authored Oct 30, 2023
1 parent fcc1129 commit 509c23e
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 6 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
See the "API Implementations" section of the `go.opentelemetry.io/otel/trace` package documentation for more informatoin about how to accomplish this. (#4620)
- `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc` does no longer depend on `go.opentelemetry.io/otel/exporters/otlp/otlpmetric`. (#4660)
- `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp` does no longer depend on `go.opentelemetry.io/otel/exporters/otlp/otlpmetric`. (#4660)
- Retry for `502 Bad Gateway` and `504 Gateway Timeout` HTTP statuses in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp`. (#4670)
- Retry for `502 Bad Gateway` and `504 Gateway Timeout` HTTP statuses in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`. (#4670)

### Fixed

Expand Down
5 changes: 4 additions & 1 deletion exporters/otlp/otlpmetric/otlpmetrichttp/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,10 @@ func (c *client) UploadMetrics(ctx context.Context, protoMetrics *metricpb.Resou
}
}
return nil
case sc == http.StatusTooManyRequests, sc == http.StatusServiceUnavailable:
case sc == http.StatusTooManyRequests,
sc == http.StatusBadGateway,
sc == http.StatusServiceUnavailable,
sc == http.StatusGatewayTimeout:
// Retry-able failure.
rErr = newResponseError(resp.Header)

Expand Down
12 changes: 10 additions & 2 deletions exporters/otlp/otlpmetric/otlpmetrichttp/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,9 @@ func TestConfig(t *testing.T) {

t.Run("WithRetry", func(t *testing.T) {
emptyErr := errors.New("")
rCh := make(chan otest.ExportResult, 3)
rCh := make(chan otest.ExportResult, 5)
header := http.Header{http.CanonicalHeaderKey("Retry-After"): {"10"}}
// Both retryable errors.
// All retryable errors.
rCh <- otest.ExportResult{Err: &otest.HTTPResponseError{
Status: http.StatusServiceUnavailable,
Err: emptyErr,
Expand All @@ -139,6 +139,14 @@ func TestConfig(t *testing.T) {
Status: http.StatusTooManyRequests,
Err: emptyErr,
}}
rCh <- otest.ExportResult{Err: &otest.HTTPResponseError{
Status: http.StatusGatewayTimeout,
Err: emptyErr,
}}
rCh <- otest.ExportResult{Err: &otest.HTTPResponseError{
Status: http.StatusBadGateway,
Err: emptyErr,
}}
rCh <- otest.ExportResult{}
exp, coll := factoryFunc("", rCh, WithRetry(RetryConfig{
Enabled: true,
Expand Down
5 changes: 4 additions & 1 deletion exporters/otlp/otlptrace/otlptracehttp/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,10 @@ func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc
}
return nil

case sc == http.StatusTooManyRequests, sc == http.StatusServiceUnavailable:
case sc == http.StatusTooManyRequests,
sc == http.StatusBadGateway,
sc == http.StatusServiceUnavailable,
sc == http.StatusGatewayTimeout:
// Retry-able failures. Drain the body to reuse the connection.
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
otel.Handle(err)
Expand Down
4 changes: 2 additions & 2 deletions exporters/otlp/otlptrace/otlptracehttp/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func TestEndToEnd(t *testing.T) {
}),
},
mcCfg: mockCollectorConfig{
InjectHTTPStatus: []int{503, 503},
InjectHTTPStatus: []int{503, 502},
},
},
{
Expand All @@ -110,7 +110,7 @@ func TestEndToEnd(t *testing.T) {
}),
},
mcCfg: mockCollectorConfig{
InjectHTTPStatus: []int{503},
InjectHTTPStatus: []int{504},
InjectResponseHeader: []map[string]string{
{"Retry-After": "10"},
},
Expand Down

0 comments on commit 509c23e

Please sign in to comment.