feat: HTTPConnectionManager settings for XFF headers on ClientTraffic…

…Policy (#2252)

Signed-off-by: David Alger <davidmalger@gmail.com>

api/v1alpha1/clienttrafficpolicy_types.go

@@ -66,6 +66,10 @@ type ClientTrafficPolicySpec struct {
 	//
 	// +optional
 	EnableProxyProtocol *bool `json:"enableProxyProtocol,omitempty"`
+	// HTTPConnectionManager provides HTTP Connection Manager configuration on the listener.
+	//
+	// +optional
+	HTTPConnectionManager *HTTPConnectionManagerSettings `json:"httpConnectionManager,omitempty"`
 	// HTTP3 provides HTTP/3 configuration on the listener.
 	//
 	// +optional
@@ -84,6 +88,24 @@ type ClientTrafficPolicySpec struct {
 	HTTP1 *HTTP1Settings `json:"http1,omitempty"`
 }
 
+// HTTPConnectionManagerSettings provides HTTP Connection Manager configuration on the listener.
+type HTTPConnectionManagerSettings struct {
+	// UseRemoteAddress controls whether the connection manager will use the real remote address of the
+	// client connection when determining internal versus external origin and manipulating various headers.
+	// Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+	// Enabled by default.
+	//
+	// +optional
+	// +kubebuilder:default=true
+	UseRemoteAddress *bool `json:"useRemoteAddress,omitempty"`
+	// XffNumTrustedHops controls the number of additional ingress proxy hops from the right side of XFF HTTP
+	// headers to trust when determining the origin client's IP address.
+	// Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+	//
+	// +optional
+	XffNumTrustedHops *uint32 `json:"xffNumTrustedHops,omitempty"`
+}
+
 // HTTP3Settings provides HTTP/3 configuration on the listener.
 type HTTP3Settings struct {
 }

charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml

@@ -66,6 +66,26 @@ spec:
               http3:
                 description: HTTP3 provides HTTP/3 configuration on the listener.
                 type: object
+              httpConnectionManager:
+                description: HTTPConnectionManager provides HTTP Connection Manager
+                  configuration on the listener.
+                properties:
+                  useRemoteAddress:
+                    default: true
+                    description: UseRemoteAddress controls whether the connection
+                      manager will use the real remote address of the client connection
+                      when determining internal versus external origin and manipulating
+                      various headers. Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+                      Enabled by default.
+                    type: boolean
+                  xffNumTrustedHops:
+                    description: XffNumTrustedHops controls the number of additional
+                      ingress proxy hops from the right side of XFF HTTP headers to
+                      trust when determining the origin client's IP address. Refer
+                      to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+                    format: int32
+                    type: integer
+                type: object
               path:
                 description: Path enables managing how the incoming path set by clients
                   can be normalized.

internal/gatewayapi/clienttrafficpolicy.go

@@ -292,6 +292,9 @@ func (t *Translator) translateClientTrafficPolicyForListener(policySpec *egv1a1.
 		// Translate Proxy Protocol
 		translateListenerProxyProtocol(policySpec.EnableProxyProtocol, httpIR)
 
+		// Translate HTTP Connection Manager
+		translateHTTPConnectionManager(policySpec.HTTPConnectionManager, httpIR)
+
 		// Translate Suppress Envoy Headers
 		translateListenerSuppressEnvoyHeaders(policySpec.SuppressEnvoyHeaders, httpIR)
 
@@ -375,6 +378,16 @@ func translateListenerProxyProtocol(enableProxyProtocol *bool, httpIR *ir.HTTPLi
 	}
 }
 
+func translateHTTPConnectionManager(httpConnectionManager *egv1a1.HTTPConnectionManagerSettings, httpIR *ir.HTTPListener) {
+	// Return early if not set
+	if httpConnectionManager == nil {
+		return
+	}
+
+	httpIR.UseRemoteAddress = httpConnectionManager.UseRemoteAddress
+	httpIR.XffNumTrustedHops = ptr.Deref(httpConnectionManager.XffNumTrustedHops, 0)
+}
+
 func translateListenerSuppressEnvoyHeaders(suppressEnvoyHeaders *bool, httpIR *ir.HTTPListener) {
 	if suppressEnvoyHeaders != nil {
 		httpIR.SuppressEnvoyHeaders = *suppressEnvoyHeaders

internal/gatewayapi/testdata/clienttrafficpolicy-http-connection-manager-settings.in.yaml

@@ -0,0 +1,37 @@
+clientTrafficPolicies:
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: ClientTrafficPolicy
+  metadata:
+    namespace: envoy-gateway
+    name: target-gateway-1
+  spec:
+    httpConnectionManager:
+      useRemoteAddress: false
+      xffNumTrustedHops: 2
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gateway-1
+      namespace: envoy-gateway
+      sectionName: http-1
+gateways:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    namespace: envoy-gateway
+    name: gateway-1
+  spec:
+    gatewayClassName: envoy-gateway-class
+    listeners:
+    - name: http-1
+      protocol: HTTP
+      port: 80
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: http-2
+      protocol: HTTP
+      port: 8080
+      allowedRoutes:
+        namespaces:
+          from: Same

internal/gatewayapi/testdata/clienttrafficpolicy-http-connection-manager-settings.out.yaml

@@ -0,0 +1,143 @@
+clientTrafficPolicies:
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: ClientTrafficPolicy
+  metadata:
+    creationTimestamp: null
+    name: target-gateway-1
+    namespace: envoy-gateway
+  spec:
+    httpConnectionManager:
+      useRemoteAddress: false
+      xffNumTrustedHops: 2
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gateway-1
+      namespace: envoy-gateway
+      sectionName: http-1
+  status:
+    conditions:
+    - lastTransitionTime: null
+      message: ClientTrafficPolicy has been accepted.
+      reason: Accepted
+      status: "True"
+      type: Accepted
+gateways:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    creationTimestamp: null
+    name: gateway-1
+    namespace: envoy-gateway
+  spec:
+    gatewayClassName: envoy-gateway-class
+    listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      name: http-1
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      name: http-2
+      port: 8080
+      protocol: HTTP
+  status:
+    listeners:
+    - attachedRoutes: 0
+      conditions:
+      - lastTransitionTime: null
+        message: Sending translated listener configuration to the data plane
+        reason: Programmed
+        status: "True"
+        type: Programmed
+      - lastTransitionTime: null
+        message: Listener has been successfully translated
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Listener references have been resolved
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      name: http-1
+      supportedKinds:
+      - group: gateway.networking.k8s.io
+        kind: HTTPRoute
+      - group: gateway.networking.k8s.io
+        kind: GRPCRoute
+    - attachedRoutes: 0
+      conditions:
+      - lastTransitionTime: null
+        message: Sending translated listener configuration to the data plane
+        reason: Programmed
+        status: "True"
+        type: Programmed
+      - lastTransitionTime: null
+        message: Listener has been successfully translated
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Listener references have been resolved
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      name: http-2
+      supportedKinds:
+      - group: gateway.networking.k8s.io
+        kind: HTTPRoute
+      - group: gateway.networking.k8s.io
+        kind: GRPCRoute
+infraIR:
+  envoy-gateway/gateway-1:
+    proxy:
+      listeners:
+      - address: null
+        name: envoy-gateway/gateway-1/http-1
+        ports:
+        - containerPort: 10080
+          name: http-1
+          protocol: HTTP
+          servicePort: 80
+      - address: null
+        name: envoy-gateway/gateway-1/http-2
+        ports:
+        - containerPort: 8080
+          name: http-2
+          protocol: HTTP
+          servicePort: 8080
+      metadata:
+        labels:
+          gateway.envoyproxy.io/owning-gateway-name: gateway-1
+          gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway
+      name: envoy-gateway/gateway-1
+xdsIR:
+  envoy-gateway/gateway-1:
+    accessLog:
+      text:
+      - path: /dev/stdout
+    http:
+    - address: 0.0.0.0
+      hostnames:
+      - '*'
+      isHTTP2: false
+      name: envoy-gateway/gateway-1/http-1
+      path:
+        escapedSlashesAction: UnescapeAndRedirect
+        mergeSlashes: true
+      port: 10080
+      useRemoteAddress: false
+      xffNumTrustedHops: 2
+    - address: 0.0.0.0
+      hostnames:
+      - '*'
+      isHTTP2: false
+      name: envoy-gateway/gateway-1/http-2
+      path:
+        escapedSlashesAction: UnescapeAndRedirect
+        mergeSlashes: true
+      port: 8080

internal/ir/xds.go

@@ -214,6 +214,14 @@ type HTTPListener struct {
 	SuppressEnvoyHeaders bool `json:"suppressEnvoyHeaders,omitempty" yaml:"suppressEnvoyHeaders,omitempty"`
 	// EnableProxyProtocol enables the listener to interpret proxy protocol header
 	EnableProxyProtocol bool `json:"enableProxyProtocol,omitempty" yaml:"enableProxyProtocol,omitempty"`
+	// UseRemoteAddress controls whether the connection manager will use the real remote address of the
+	// client connection when determining internal versus external origin and manipulating various headers.
+	// Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+	UseRemoteAddress *bool `json:"useRemoteAddress,omitempty" yaml:"useRemoteAddress,omitempty"`
+	// XffNumTrustedHops controls the number of additional ingress proxy hops from the right side of XFF HTTP
+	// headers to trust when determining the origin client's IP address.
+	// Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+	XffNumTrustedHops uint32 `json:"xffNumTrustedHops,omitempty" yaml:"xffNumTrustedHops,omitempty"`
 	// HTTP3 provides HTTP/3 configuration on the listener.
 	// +optional
 	HTTP3 *HTTP3Settings `json:"http3,omitempty"`
@@ -340,6 +348,8 @@ type PathSettings struct {
 	EscapedSlashesAction PathEscapedSlashAction `json:"escapedSlashesAction" yaml:"escapedSlashesAction"`
 }
 
+type HTTPConnectionManagerSettings egv1a1.HTTPConnectionManagerSettings
+
 // BackendWeights stores the weights of valid and invalid backends for the route so that 500 error responses can be returned in the same proportions
 type BackendWeights struct {
 	Valid   uint32 `json:"valid" yaml:"valid"`

internal/xds/translator/listener.go

@@ -146,6 +146,16 @@ func (t *Translator) addXdsHTTPFilterChain(xdsListener *listenerv3.Listener, irL
 	} else {
 		statPrefix = "http"
 	}
+
+	// HTTP XFF configuration
+	var useRemoteAddress bool
+	if irListener.UseRemoteAddress != nil {
+		useRemoteAddress = *irListener.UseRemoteAddress
+	} else {
+		// Enable UseRemoteAddress by default when ClientTrafficPolicy is not configured
+		useRemoteAddress = true
+	}
+
 	mgr := &hcmv3.HttpConnectionManager{
 		AccessLog:  al,
 		CodecType:  hcmv3.HttpConnectionManager_AUTO,
@@ -162,7 +172,8 @@ func (t *Translator) addXdsHTTPFilterChain(xdsListener *listenerv3.Listener, irL
 		// Set it by default to also support HTTP1.1 to HTTP2 Upgrades
 		Http2ProtocolOptions: http2ProtocolOptions(),
 		// https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
-		UseRemoteAddress: &wrappers.BoolValue{Value: true},
+		UseRemoteAddress:  &wrappers.BoolValue{Value: useRemoteAddress},
+		XffNumTrustedHops: irListener.XffNumTrustedHops,
 		// normalize paths according to RFC 3986
 		NormalizePath:                &wrapperspb.BoolValue{Value: true},
 		MergeSlashes:                 irListener.Path.MergeSlashes,