add support for endpoint slices (#5745)

Adds support for EndpointSlices, gated by the feature gate "useEndpointSlices". Fixes #2696. Signed-off-by: Clayton Gonsalves <clayton.gonsalves@reddit.com>
projectcontour · Oct 24, 2023 · a216475 · a216475
1 parent bc42c50
commit a216475
Show file tree

Hide file tree

Showing 29 changed files with 2,326 additions and 28 deletions.
diff --git a/.github/workflows/build_daily.yaml b/.github/workflows/build_daily.yaml
@@ -152,3 +152,38 @@ jobs:
           steps: ${{ toJson(steps) }}
           channel: '#contour-ci-notifications'
         if: ${{ failure() && github.ref == 'refs/heads/main' }}
+  e2e-endpoint-slices:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-${{ github.job }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ github.job }}-go-
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - name: add deps to path
+        run: |
+          ./hack/actions/install-kubernetes-toolchain.sh $GITHUB_WORKSPACE/bin
+          echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH
+      - name: e2e tests
+        env:
+          CONTOUR_E2E_IMAGE: ghcr.io/projectcontour/contour:main
+          CONTOUR_E2E_USE_ENDPOINT_SLICES: true
+        run: |
+          make setup-kind-cluster run-e2e cleanup-kind
+      - uses: act10ns/slack@v2
+        with:
+          status: ${{ job.status }}
+          steps: ${{ toJson(steps) }}
+          channel: '#contour-ci-notifications'
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+
diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go
@@ -84,8 +84,19 @@ type ContourConfigurationSpec struct {
 
 	// Tracing defines properties for exporting trace data to OpenTelemetry.
 	Tracing *TracingConfig `json:"tracing,omitempty"`
+
+	// FeatureFlags defines toggle to enable new contour features.
+	// Available toggles are:
+	// useEndpointSlices - configures contour to fetch endpoint data
+	// from k8s endpoint slices. defaults to false and reading endpoint
+	// data from the k8s endpoints.
+	FeatureFlags FeatureFlags `json:"featureFlags,omitempty"`
 }
 
+// FeatureFlags defines the set of feature flags
+// to toggle new contour features.
+type FeatureFlags []string
+
 // XDSServerType is the type of xDS server implementation.
 type XDSServerType string
 

diff --git a/apis/projectcontour/v1alpha1/contourconfig_helpers.go b/apis/projectcontour/v1alpha1/contourconfig_helpers.go
@@ -17,9 +17,18 @@ import (
 	"fmt"
 	"strconv"
 
+	"golang.org/x/exp/slices"
 	"k8s.io/apimachinery/pkg/util/sets"
 )
 
+const (
+	featureFlagUseEndpointSlices string = "useEndpointSlices"
+)
+
+var featureFlagsMap = map[string]bool{
+	featureFlagUseEndpointSlices: true,
+}
+
 // Validate configuration that is not already covered by CRD validation.
 func (c *ContourConfigurationSpec) Validate() error {
 	// Validation of root configuration fields.
@@ -215,6 +224,20 @@ func (e *EnvoyTLS) SanitizedCipherSuites() []string {
 	return validatedCiphers
 }
 
+func (f FeatureFlags) Validate() error {
+	for _, featureFlag := range f {
+		if _, found := featureFlagsMap[featureFlag]; !found {
+			return fmt.Errorf("invalid contour configuration, unknown feature flag:%s", featureFlag)
+		}
+	}
+
+	return nil
+}
+
+func (f FeatureFlags) IsEndpointSliceEnabled() bool {
+	return slices.Contains(f, featureFlagUseEndpointSlices)
+}
+
 // Validate ensures that exactly one of ControllerName or GatewayRef are specified.
 func (g *GatewayConfig) Validate() error {
 	if g == nil {

diff --git a/apis/projectcontour/v1alpha1/contourconfig_helpers_test.go b/apis/projectcontour/v1alpha1/contourconfig_helpers_test.go
@@ -14,6 +14,7 @@
 package v1alpha1_test
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/projectcontour/contour/apis/projectcontour/v1alpha1"
@@ -294,3 +295,39 @@ func TestAccessLogFormatExtensions(t *testing.T) {
 	}
 	assert.Empty(t, e3.AccessLogFormatterExtensions())
 }
+
+func TestFeatureFlagsValidate(t *testing.T) {
+	tests := []struct {
+		name     string
+		flags    v1alpha1.FeatureFlags
+		expected error
+	}{
+		{
+			name:     "valid flag",
+			flags:    v1alpha1.FeatureFlags{"useEndpointSlices"},
+			expected: nil,
+		},
+		{
+			name:     "invalid flag",
+			flags:    v1alpha1.FeatureFlags{"invalidFlag"},
+			expected: fmt.Errorf("invalid contour configuration, unknown feature flag:invalidFlag"),
+		},
+		{
+			name:     "mix of valid and invalid flags",
+			flags:    v1alpha1.FeatureFlags{"useEndpointSlices", "invalidFlag"},
+			expected: fmt.Errorf("invalid contour configuration, unknown feature flag:invalidFlag"),
+		},
+		{
+			name:     "empty flags",
+			flags:    v1alpha1.FeatureFlags{},
+			expected: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.flags.Validate()
+			assert.Equal(t, tt.expected, err)
+		})
+	}
+}
diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go
diff --git a/changelogs/unreleased/5745-clayton-gonsalves-minor.md b/changelogs/unreleased/5745-clayton-gonsalves-minor.md
@@ -0,0 +1,6 @@
+## Add Kubernetes Endpoint Slice support
+
+This change optionally enables Contour to consume the kubernetes endpointslice API to determine the endpoints to configure Envoy with.
+Note: This change is off by default and is gated by the feature flag `useEndpointSlices`.
+
+This feature will be enabled by default in a future version on Contour once it has had sufficient bake time in production environments.
diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go
@@ -64,6 +64,7 @@ import (
 	"github.com/projectcontour/contour/internal/xdscache"
 	xdscache_v3 "github.com/projectcontour/contour/internal/xdscache/v3"
 	"github.com/projectcontour/contour/pkg/config"
+	discoveryv1 "k8s.io/api/discovery/v1"
 )
 
 const (
@@ -190,6 +191,12 @@ type Server struct {
 	handlerCacheSyncs []cache.InformerSynced
 }
 
+type EndpointsTranslator interface {
+	cache.ResourceEventHandler
+	xdscache.ResourceCache
+	SetObserver(observer contour.Observer)
+}
+
 // NewServer returns a Server object which contains the initial configuration
 // objects required to start an instance of Contour.
 func NewServer(log logrus.FieldLogger, ctx *serveContext) (*Server, error) {
@@ -456,9 +463,13 @@ func (s *Server) doServe() error {
 
 	contourMetrics := metrics.NewMetrics(s.registry)
 
-	// Endpoints updates are handled directly by the EndpointsTranslator
-	// due to their high update rate and their orthogonal nature.
-	endpointHandler := xdscache_v3.NewEndpointsTranslator(s.log.WithField("context", "endpointstranslator"))
+	// Endpoints updates are handled directly by the EndpointsTranslator/EndpointSliceTranslator due to the high update volume.
+	var endpointHandler EndpointsTranslator
+	if contourConfiguration.FeatureFlags.IsEndpointSliceEnabled() {
+		endpointHandler = xdscache_v3.NewEndpointSliceTranslator(s.log.WithField("context", "endpointslicetranslator"))
+	} else {
+		endpointHandler = xdscache_v3.NewEndpointsTranslator(s.log.WithField("context", "endpointstranslator"))
+	}
 
 	resources := []xdscache.ResourceCache{
 		xdscache_v3.NewListenerCache(listenerConfig, *contourConfiguration.Envoy.Metrics, *contourConfiguration.Envoy.Health, *contourConfiguration.Envoy.Network.EnvoyAdminPort),
@@ -475,7 +486,7 @@ func (s *Server) doServe() error {
 	snapshotHandler := xdscache.NewSnapshotHandler(resources, s.log.WithField("context", "snapshotHandler"))
 
 	// register observer for endpoints updates.
-	endpointHandler.Observer = contour.ComposeObservers(snapshotHandler)
+	endpointHandler.SetObserver(contour.ComposeObservers(snapshotHandler))
 
 	// Log that we're using the fallback certificate if configured.
 	if contourConfiguration.HTTPProxy.FallbackCertificate != nil {
@@ -608,12 +619,21 @@ func (s *Server) doServe() error {
 		s.log.WithError(err).WithField("resource", "secrets").Fatal("failed to create informer")
 	}
 
-	// Inform on endpoints.
-	if err := s.informOnResource(&corev1.Endpoints{}, &contour.EventRecorder{
-		Next:    endpointHandler,
-		Counter: contourMetrics.EventHandlerOperations,
-	}); err != nil {
-		s.log.WithError(err).WithField("resource", "endpoints").Fatal("failed to create informer")
+	// Inform on endpoints/endpointSlices.
+	if contourConfiguration.FeatureFlags.IsEndpointSliceEnabled() {
+		if err := s.informOnResource(&discoveryv1.EndpointSlice{}, &contour.EventRecorder{
+			Next:    endpointHandler,
+			Counter: contourMetrics.EventHandlerOperations,
+		}); err != nil {
+			s.log.WithError(err).WithField("resource", "endpointslices").Fatal("failed to create informer")
+		}
+	} else {
+		if err := s.informOnResource(&corev1.Endpoints{}, &contour.EventRecorder{
+			Next:    endpointHandler,
+			Counter: contourMetrics.EventHandlerOperations,
+		}); err != nil {
+			s.log.WithError(err).WithField("resource", "endpoints").Fatal("failed to create informer")
+		}
 	}
 
 	// Register our event handler with the manager.

diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go
@@ -590,6 +590,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 		Policy:                      policy,
 		Metrics:                     &contourMetrics,
 		Tracing:                     tracingConfig,
+		FeatureFlags:                ctx.Config.FeatureFlags,
 	}
 
 	xdsServerType := contour_api_v1alpha1.ContourServerType

diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml
@@ -471,6 +471,14 @@ spec:
                         type: string
                     type: object
                 type: object
+              featureFlags:
+                description: 'FeatureFlags defines toggle to enable new contour features.
+                  Available toggles are: useEndpointSlices - configures contour to
+                  fetch endpoint data from k8s endpoint slices. defaults to false
+                  and reading endpoint data from the k8s endpoints.'
+                items:
+                  type: string
+                type: array
               gateway:
                 description: Gateway contains parameters for the gateway-api Gateway
                   that Contour is configured to serve traffic.
@@ -3945,6 +3953,14 @@ spec:
                             type: string
                         type: object
                     type: object
+                  featureFlags:
+                    description: 'FeatureFlags defines toggle to enable new contour
+                      features. Available toggles are: useEndpointSlices - configures
+                      contour to fetch endpoint data from k8s endpoint slices. defaults
+                      to false and reading endpoint data from the k8s endpoints.'
+                    items:
+                      type: string
+                    type: array
                   gateway:
                     description: Gateway contains parameters for the gateway-api Gateway
                       that Contour is configured to serve traffic.

diff --git a/examples/contour/02-role-contour.yaml b/examples/contour/02-role-contour.yaml
@@ -18,6 +18,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - gateway.networking.k8s.io
   resources:

diff --git a/examples/gateway-provisioner/01-roles.yaml b/examples/gateway-provisioner/01-roles.yaml
@@ -59,6 +59,14 @@ rules:
   - create
   - get
   - update
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - gateway.networking.k8s.io
   resources:

diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml
@@ -690,6 +690,14 @@ spec:
                         type: string
                     type: object
                 type: object
+              featureFlags:
+                description: 'FeatureFlags defines toggle to enable new contour features.
+                  Available toggles are: useEndpointSlices - configures contour to
+                  fetch endpoint data from k8s endpoint slices. defaults to false
+                  and reading endpoint data from the k8s endpoints.'
+                items:
+                  type: string
+                type: array
               gateway:
                 description: Gateway contains parameters for the gateway-api Gateway
                   that Contour is configured to serve traffic.
@@ -4164,6 +4172,14 @@ spec:
                             type: string
                         type: object
                     type: object
+                  featureFlags:
+                    description: 'FeatureFlags defines toggle to enable new contour
+                      features. Available toggles are: useEndpointSlices - configures
+                      contour to fetch endpoint data from k8s endpoint slices. defaults
+                      to false and reading endpoint data from the k8s endpoints.'
+                    items:
+                      type: string
+                    type: array
                   gateway:
                     description: Gateway contains parameters for the gateway-api Gateway
                       that Contour is configured to serve traffic.
@@ -8435,6 +8451,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - gateway.networking.k8s.io
   resources: