Skip to content

Commit

Permalink
Fix flaky OTLP exporter reconnect test (open-telemetry#1814)
Browse files Browse the repository at this point in the history
* Fix flaky OTLP exporter reconnect test

The tests that check the OTLP exporter will reconnect wait for the
reconnect loop to complete, in theory. However, they do not yield the
active goroutine scheduling. The reconnect loop is in its own goroutine
meaning it is unlikely for that loop to be re-entered, especially on
slow systems. This updates the tests call runtime.Gosched when waiting
for the reconnect loop and yield the scheduling to other goroutines.

* Add changes to changelog

* Use time.After instead of Timer

* Remove changelog entry
  • Loading branch information
MrAlias authored Apr 15, 2021
1 parent b09df84 commit d616df6
Showing 1 changed file with 36 additions and 5 deletions.
41 changes: 36 additions & 5 deletions exporters/otlp/otlpgrpc/otlp_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"
"net"
"runtime"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -144,7 +145,7 @@ func TestNewExporter_invokeStartThenStopManyTimes(t *testing.T) {
func TestNewExporter_collectorConnectionDiesThenReconnectsWhenInRestMode(t *testing.T) {
mc := runMockCollector(t)

reconnectionPeriod := 2 * time.Second // 2 second + jitter rest time after reconnection
reconnectionPeriod := 20 * time.Millisecond
ctx := context.Background()
exp := newGRPCExporter(t, ctx, mc.endpoint,
otlpgrpc.WithReconnectionPeriod(reconnectionPeriod))
Expand All @@ -165,8 +166,18 @@ func TestNewExporter_collectorConnectionDiesThenReconnectsWhenInRestMode(t *test
mc.endpoint,
)

// give it time for first reconnection
<-time.After(time.Millisecond * 20)
// Give the exporter sometime to reconnect
func() {
timer := time.After(reconnectionPeriod * 10)
for {
select {
case <-timer:
return
default:
runtime.Gosched()
}
}
}()

// second export, it will detect connection issue, change state of exporter to disconnected and
// send message to disconnected channel but this time reconnection gouroutine will be in (rest mode, not listening to the disconnected channel)
Expand All @@ -184,7 +195,17 @@ func TestNewExporter_collectorConnectionDiesThenReconnectsWhenInRestMode(t *test

// make sure reconnection loop hits beginning and goes back to waiting mode
// after hitting beginning of the loop it should reconnect
<-time.After(time.Second * 4)
func() {
timer := time.After(reconnectionPeriod * 10)
for {
select {
case <-timer:
return
default:
runtime.Gosched()
}
}
}()

n := 10
for i := 0; i < n; i++ {
Expand Down Expand Up @@ -240,7 +261,17 @@ func TestNewExporter_collectorConnectionDiesThenReconnects(t *testing.T) {
nmc := runMockCollectorAtEndpoint(t, mc.endpoint)

// Give the exporter sometime to reconnect
<-time.After(reconnectionPeriod * 4)
func() {
timer := time.After(reconnectionPeriod * 10)
for {
select {
case <-timer:
return
default:
runtime.Gosched()
}
}
}()

n := 10
for i := 0; i < n; i++ {
Expand Down

0 comments on commit d616df6

Please sign in to comment.