Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

xds: deflake Test/ServerSideXDS_ServingModeChanges #4689

Merged
merged 5 commits into from
Aug 24, 2021
Merged

xds: deflake Test/ServerSideXDS_ServingModeChanges #4689

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d5a4781
xds: accept IDLE and READY as cc state in test
easwars Aug 19, 2021
3241f1e
skip checks on clientconn state
easwars Aug 23, 2021
4791af7
wait for rpcs to start failing
easwars Aug 23, 2021
0423cf0
take testing.T as input to waitForXxxRPC functions
easwars Aug 23, 2021
330fed8
fix typo s/cc2/cc1
easwars Aug 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 38 additions & 45 deletions xds/internal/test/xds_server_serving_mode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,10 @@ import (
"net"
"sync"
"testing"
"time"

v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"

"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
xdscreds "google.golang.org/grpc/credentials/xds"
"google.golang.org/grpc/internal/testutils"
Expand Down Expand Up @@ -159,28 +158,20 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
}

// Create a ClientConn to the first listener and make a successful RPCs.
cc1, err := grpc.DialContext(ctx, lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
cc1, err := grpc.Dial(lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to dial local test server: %v", err)
}
defer cc1.Close()

client1 := testpb.NewTestServiceClient(cc1)
if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
waitForSuccessfulRPC(ctx, t, cc1)

// Create a ClientConn to the second listener and make a successful RPCs.
cc2, err := grpc.DialContext(ctx, lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
cc2, err := grpc.Dial(lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to dial local test server: %v", err)
}
defer cc2.Close()

client2 := testpb.NewTestServiceClient(cc2)
if _, err := client2.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
waitForSuccessfulRPC(ctx, t, cc2)

// Update the management server to remove the second listener resource. This
// should push only the second listener into "not-serving" mode.
Expand All @@ -194,21 +185,9 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
t.Fatal(err)
}

// Make sure cc1 is still in READY state, while cc2 has moved out of READY.
if s := cc1.GetState(); s != connectivity.Ready {
t.Fatalf("clientConn1 state is %s, want %s", s, connectivity.Ready)
}
if !cc2.WaitForStateChange(ctx, connectivity.Ready) {
t.Fatal("clientConn2 failed to move out of READY")
}

// Make sure RPCs succeed on cc1 and fail on cc2.
if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
if _, err := client2.EmptyCall(ctx, &testpb.Empty{}); err == nil {
t.Fatal("rpc EmptyCall() succeeded when expected to fail")
}
waitForSuccessfulRPC(ctx, t, cc1)
waitForFailedRPC(ctx, t, cc2)

// Update the management server to remove the first listener resource as
// well. This should push the first listener into "not-serving" mode. Second
Expand All @@ -223,18 +202,9 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
t.Fatal(err)
}

// Make sure cc1 has moved out of READY.
if !cc1.WaitForStateChange(ctx, connectivity.Ready) {
t.Fatal("clientConn1 failed to move out of READY")
}

// Make sure RPCs fail on both.
if _, err := client1.EmptyCall(ctx, &testpb.Empty{}); err == nil {
t.Fatal("rpc EmptyCall() succeeded when expected to fail")
}
if _, err := client2.EmptyCall(ctx, &testpb.Empty{}); err == nil {
t.Fatal("rpc EmptyCall() succeeded when expected to fail")
}
waitForFailedRPC(ctx, t, cc1)
waitForFailedRPC(ctx, t, cc2)

// Make sure new connection attempts to "not-serving" servers fail. We use a
// short timeout since we expect this to fail.
Expand All @@ -261,12 +231,8 @@ func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) {
}

// The clientConns created earlier should be able to make RPCs now.
if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
if _, err := client2.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
waitForSuccessfulRPC(ctx, t, cc1)
waitForSuccessfulRPC(ctx, t, cc2)
}

func waitForModeChange(ctx context.Context, modeTracker *modeTracker, addr net.Addr, wantMode xds.ServingMode) error {
Expand All @@ -279,3 +245,30 @@ func waitForModeChange(ctx context.Context, modeTracker *modeTracker, addr net.A
}
}
}

func waitForSuccessfulRPC(ctx context.Context, t *testing.T, cc *grpc.ClientConn) {
t.Helper()

c := testpb.NewTestServiceClient(cc)
if _, err := c.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
}

func waitForFailedRPC(ctx context.Context, t *testing.T, cc *grpc.ClientConn) {
t.Helper()

c := testpb.NewTestServiceClient(cc)
ticker := time.NewTimer(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
t.Fatalf("failure when waiting for RPCs to fail: %v", ctx.Err())
case <-ticker.C:
if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
return
}
}
}
}