Skip to content

Commit

Permalink
fix flaky e2e test
Browse files Browse the repository at this point in the history 
Signed-off-by: Clayton Gonsalves <clayton.gonsalves@reddit.com>
  • Loading branch information
@clayton-gonsalves
clayton-gonsalves committed Sep 19, 2023
1 parent b1993b2 commit 03a4e3d
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 254 deletions.
238 changes: 0 additions & 238 deletions internal/xdscache/v3/endpointslicetranslator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,10 @@ import (
"github.com/projectcontour/contour/internal/fixture"
"github.com/projectcontour/contour/internal/protobuf"
"github.com/projectcontour/contour/internal/ref"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/types"
)

func TestEndpointSliceTranslatorContents(t *testing.T) {
Expand Down Expand Up @@ -1198,239 +1196,3 @@ func TestEndpointSliceTranslatorDefaultWeightedService(t *testing.T) {

protobuf.ExpectEqual(t, want, endpointSliceTranslator.Contents())
}

func TestEndpointSliceTranslatorWithDuplicateEndpoints(t *testing.T) {
clusters := []*dag.ServiceCluster{
{
ClusterName: "default/httpbin-org/a",
Services: []dag.WeightedService{
{
Weight: 1,
ServiceName: "httpbin-org",
ServiceNamespace: "default",
ServicePort: v1.ServicePort{Name: "a"},
},
},
},
{
ClusterName: "default/httpbin-org/b",
Services: []dag.WeightedService{
{
Weight: 1,
ServiceName: "httpbin-org",
ServiceNamespace: "default",
ServicePort: v1.ServicePort{Name: "b"},
},
},
},
{
ClusterName: "default/simple",
Services: []dag.WeightedService{
{
Weight: 1,
ServiceName: "simple",
ServiceNamespace: "default",
ServicePort: v1.ServicePort{},
},
},
},
{
ClusterName: "default/healthcheck-port",
Services: []dag.WeightedService{
{
Weight: 1,
ServiceName: "healthcheck-port",
ServiceNamespace: "default",
ServicePort: v1.ServicePort{Name: "a"},
HealthPort: v1.ServicePort{Name: "health", Port: 8998},
},
},
},
}

tests := map[string]struct {
endpointSlice *discoveryv1.EndpointSlice
existingEndpointsCache map[types.NamespacedName]map[string]*discoveryv1.EndpointSlice
want []proto.Message
wantUpdate bool
}{
"simple": {
endpointSlice: endpointSlice("default", "simple-eps-fs9du", "simple", discoveryv1.AddressTypeIPv4, []discoveryv1.Endpoint{
{
Addresses: []string{"192.168.183.24"},
},
}, []discoveryv1.EndpointPort{
{
Port: ref.To[int32](8080),
Protocol: ref.To[v1.Protocol]("TCP"),
},
},
),
existingEndpointsCache: map[types.NamespacedName]map[string]*discoveryv1.EndpointSlice{
{Namespace: "default", Name: "simple"}: {
"simple-eps-9s8dd": endpointSlice("default", "simple-eps-fs9du", "simple", discoveryv1.AddressTypeIPv4, []discoveryv1.Endpoint{
{
Addresses: []string{"192.168.183.24"},
},
}, []discoveryv1.EndpointPort{
{
Port: ref.To[int32](8080),
Protocol: ref.To[v1.Protocol]("TCP"),
},
},
),
},
},
want: []proto.Message{
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/healthcheck-port"},
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/httpbin-org/a"},
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/httpbin-org/b"},
&envoy_endpoint_v3.ClusterLoadAssignment{
ClusterName: "default/simple",
Endpoints: envoy_v3.WeightedEndpoints(1, envoy_v3.SocketAddress("192.168.183.24", 8080)),
},
},
wantUpdate: true,
},
"multiple slices": {
endpointSlice: endpointSlice("default", "simple-eps-fs9du", "simple", discoveryv1.AddressTypeIPv4, []discoveryv1.Endpoint{
{
Addresses: []string{
"50.17.206.192",
},
},
{
Addresses: []string{
"23.23.247.89",
},
},
{
Addresses: []string{
"50.17.192.147",
},
},
{
Addresses: []string{
"50.19.99.160",
},
},
}, []discoveryv1.EndpointPort{
{
Port: ref.To[int32](80),
Protocol: ref.To[v1.Protocol]("TCP"),
},
},
),
existingEndpointsCache: map[types.NamespacedName]map[string]*discoveryv1.EndpointSlice{
{Namespace: "default", Name: "simple"}: {
"simple-eps-9s8dd": endpointSlice("default", "simple-eps-fs9du", "simple", discoveryv1.AddressTypeIPv4, []discoveryv1.Endpoint{
{
Addresses: []string{"192.168.183.24"},
},
{
Addresses: []string{"23.23.247.89"},
},
}, []discoveryv1.EndpointPort{
{
Port: ref.To[int32](80),
Protocol: ref.To[v1.Protocol]("TCP"),
},
},
),
},
},
want: []proto.Message{
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/healthcheck-port"},
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/httpbin-org/a"},
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/httpbin-org/b"},
&envoy_endpoint_v3.ClusterLoadAssignment{
ClusterName: "default/simple",
Endpoints: envoy_v3.WeightedEndpoints(1,
envoy_v3.SocketAddress("192.168.183.24", 80),
envoy_v3.SocketAddress("23.23.247.89", 80),
envoy_v3.SocketAddress("50.17.192.147", 80),
envoy_v3.SocketAddress("50.17.206.192", 80),
envoy_v3.SocketAddress("50.19.99.160", 80),
),
},
},
wantUpdate: true,
},
"multiple ports": {
endpointSlice: endpointSlice("default", "httpbin-org-s9d8f", "httpbin-org", discoveryv1.AddressTypeIPv4, []discoveryv1.Endpoint{
{
Addresses: []string{"10.10.1.1"},
},
}, []discoveryv1.EndpointPort{
{
Name: ref.To[string]("a"),
Port: ref.To[int32](8675),
Protocol: ref.To[v1.Protocol]("TCP"),
},
{
Name: ref.To[string]("b"),
Port: ref.To[int32](309),
Protocol: ref.To[v1.Protocol]("TCP"),
},
},
),
existingEndpointsCache: map[types.NamespacedName]map[string]*discoveryv1.EndpointSlice{
{Namespace: "default", Name: "httpbin-org"}: {
"httpbin-org-std8f": endpointSlice("default", "httpbin-org-std8f", "httpbin-org", discoveryv1.AddressTypeIPv4, []discoveryv1.Endpoint{
{
Addresses: []string{"10.10.1.1"},
},
{
Addresses: []string{"10.10.3.3"},
},
}, []discoveryv1.EndpointPort{
{
Name: ref.To[string]("a"),
Port: ref.To[int32](8675),
Protocol: ref.To[v1.Protocol]("TCP"),
},
{
Name: ref.To[string]("b"),
Port: ref.To[int32](309),
Protocol: ref.To[v1.Protocol]("TCP"),
},
},
),
},
},
want: []proto.Message{
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/healthcheck-port"},
&envoy_endpoint_v3.ClusterLoadAssignment{
ClusterName: "default/httpbin-org/a",
Endpoints: envoy_v3.WeightedEndpoints(1,
envoy_v3.SocketAddress("10.10.1.1", 8675),
envoy_v3.SocketAddress("10.10.3.3", 8675),
),
},
&envoy_endpoint_v3.ClusterLoadAssignment{
ClusterName: "default/httpbin-org/b",
Endpoints: envoy_v3.WeightedEndpoints(1,
envoy_v3.SocketAddress("10.10.1.1", 309),
envoy_v3.SocketAddress("10.10.3.3", 309),
),
},
&envoy_endpoint_v3.ClusterLoadAssignment{ClusterName: "default/simple"},
},
wantUpdate: true,
},
}

for name, tc := range tests {
t.Run(name, func(t *testing.T) {
endpointSliceTranslator := NewEndpointSliceTranslator(fixture.NewTestLogger(t))
observer := &simpleObserver{}
endpointSliceTranslator.Observer = observer
require.NoError(t, endpointSliceTranslator.cache.SetClusters(clusters))
endpointSliceTranslator.cache.endpointSlices = tc.existingEndpointsCache
endpointSliceTranslator.OnAdd(tc.endpointSlice, false)
got := endpointSliceTranslator.Contents()
assert.ElementsMatch(t, tc.want, got)
require.Equal(t, tc.wantUpdate, observer.updated)
})
}
}
26 changes: 17 additions & 9 deletions test/e2e/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"io"
"os"
"strconv"
"time"

"github.com/onsi/ginkgo/v2"
contour_api_v1alpha1 "github.com/projectcontour/contour/apis/projectcontour/v1alpha1"
Expand Down Expand Up @@ -184,20 +185,27 @@ func (e *Echo) DeployN(ns, name string, replicas int32) (func(), *appsv1.Deploym
}, deployment
}

func (e *Echo) ScaleDeployment(name, ns string, replicas int32) {

func (e *Echo) ScaleAndWaitDeployment(name, ns string, replicas int32) {
deployment := &appsv1.Deployment{}
key := types.NamespacedName{
Namespace: ns,
Name: name,
}

require.NoError(
e.t,
e.client.Get(context.TODO(), types.NamespacedName{
Namespace: ns,
Name: name,
}, deployment))
require.NoError(e.t, e.client.Get(context.TODO(), key, deployment))

deployment.Spec.Replicas = &replicas

require.NoError(e.t, e.client.Update(context.TODO(), deployment))
updateAndWaitFor(e.t, e.client, deployment, func(d *appsv1.Deployment) bool {
err := e.client.Get(context.Background(), key, deployment)
if err != nil {
return false
}
if deployment.Status.Replicas == replicas && deployment.Status.ReadyReplicas == replicas {
return true
}
return false
}, time.Second, time.Second*10)
}

func (e *Echo) ListPodIPs(ns, name string) ([]string, error) {
Expand Down
25 changes: 25 additions & 0 deletions test/e2e/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,31 @@ func createAndWaitFor[T client.Object](t require.TestingT, client client.Client,
return obj, true
}

func updateAndWaitFor[T client.Object](t require.TestingT, client client.Client, obj T, condition func(T) bool, interval, timeout time.Duration) (T, bool) {
require.NoError(t, client.Update(context.Background(), obj))

key := types.NamespacedName{
Namespace: obj.GetNamespace(),
Name: obj.GetName(),
}

if err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, true, func(ctx context.Context) (bool, error) {
if err := client.Get(ctx, key, obj); err != nil {
// if there was an error, we want to keep
// retrying, so just return false, not an
// error.
return false, nil
}

return condition(obj), nil
}); err != nil {
// return the last response for logging/debugging purposes
return obj, false
}

return obj, true
}

// CreateHTTPProxyAndWaitFor creates the provided HTTPProxy in the Kubernetes API
// and then waits for the specified condition to be true.
func (f *Framework) CreateHTTPProxyAndWaitFor(proxy *contourv1.HTTPProxy, condition func(*contourv1.HTTPProxy) bool) (*contourv1.HTTPProxy, bool) {
Expand Down
17 changes: 10 additions & 7 deletions test/e2e/infra/endpointslice_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func testSimpleEndpointSlice(namespace string) {
}

f.CreateHTTPProxyAndWaitFor(p, e2e.HTTPProxyValid)
time.Sleep(time.Second * 5)
time.Sleep(time.Second * 10)

k8sPodIPs, err := f.Fixtures.Echo.ListPodIPs(namespace, "echo")
require.NoError(f.T(), err)
Expand All @@ -69,26 +69,29 @@ func testSimpleEndpointSlice(namespace string) {
require.ElementsMatch(f.T(), k8sPodIPs, envoyEndpoints)

// scale up to 10 pods
f.Fixtures.Echo.ScaleDeployment("echo", namespace, 10)
time.Sleep(time.Second * 5)
f.Fixtures.Echo.ScaleAndWaitDeployment("echo", namespace, 10)
// give time for changes to be propagated by envoy and contour
time.Sleep(time.Second * 10)
k8sPodIPs, err = f.Fixtures.Echo.ListPodIPs(namespace, "echo")
require.NoError(f.T(), err)
envoyEndpoints, err = GetIPsFromAdminRequest()
require.NoError(f.T(), err)
require.ElementsMatch(f.T(), k8sPodIPs, envoyEndpoints)

// scale down to 2 pods
f.Fixtures.Echo.ScaleDeployment("echo", namespace, 2)
time.Sleep(time.Second * 5)
f.Fixtures.Echo.ScaleAndWaitDeployment("echo", namespace, 2)
// give time for changes to be propagated by envoy and contour
time.Sleep(time.Second * 10)
k8sPodIPs, err = f.Fixtures.Echo.ListPodIPs(namespace, "echo")
require.NoError(f.T(), err)
envoyEndpoints, err = GetIPsFromAdminRequest()
require.NoError(f.T(), err)
require.ElementsMatch(f.T(), k8sPodIPs, envoyEndpoints)

// scale to 0
f.Fixtures.Echo.ScaleDeployment("echo", namespace, 0)
time.Sleep(time.Second * 5)
f.Fixtures.Echo.ScaleAndWaitDeployment("echo", namespace, 0)
// give time for changes to be propagated by envoy and contour
time.Sleep(time.Second * 10)
k8sPodIPs, err = f.Fixtures.Echo.ListPodIPs(namespace, "echo")
require.NoError(f.T(), err)
envoyEndpoints, err = GetIPsFromAdminRequest()
Expand Down

0 comments on commit 03a4e3d

Please sign in to comment.