From a4983658680d6520888f700e132ac5e8b7bb1c5d Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 2 Aug 2023 10:49:55 -0700 Subject: [PATCH] [release/v0.5] Cherry-pick fixes needed for v0.5 (#1749) * refactor: set defaults in Deployment, else k8s sets them for you, creating infinite reconciliation loop (#1594) * fix: envoy proxy resource apply bug. Signed-off-by: qicz * update pointer. Signed-off-by: qicz * add comment Signed-off-by: qicz * update cm cmp logic. Signed-off-by: qicz * fix lint Signed-off-by: qicz * add probe field default value. Signed-off-by: qicz * fix uts Signed-off-by: qicz * align probe Signed-off-by: qicz * optimize deploy compare logic Signed-off-by: qicz * add compare deploy uts Signed-off-by: qicz * rm cm binarydata cmp Signed-off-by: qicz * rm deploy cmp logic Signed-off-by: qicz * fix ut Signed-off-by: qicz * fix lint Signed-off-by: qicz --------- Signed-off-by: qicz Signed-off-by: qi (cherry picked from commit 9ba9103880e87fac43e989909ccf550877f3bb13) * DeepCopy resources that require status updates (#1723) * Was seeing constant churn between provider runner publishing resources and gateway-api runner receiving them. * Tried to debug it by printing the o/p of `cmp.Diff` between current and previous values ``` diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index 050394ba..50d09f6f 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -8,6 +8,7 @@ package runner import ( "context" + "github.com/google/go-cmp/cmp" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/gateway-api/apis/v1beta1" "sigs.k8s.io/yaml" @@ -49,6 +50,7 @@ func (r *Runner) Start(ctx context.Context) error { } func (r *Runner) subscribeAndTranslate(ctx context.Context) { + prev := &gatewayapi.Resources{} message.HandleSubscription(r.ProviderResources.GatewayAPIResources.Subscribe(ctx), func(update message.Update[string, *gatewayapi.Resources]) { val := update.Value @@ -56,6 +58,9 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { if update.Delete || val == nil { return } + diff := cmp.Diff(prev, val) + r.Logger.WithValues("output", "diff").Info(diff) + prev = val.DeepCopy() // Translate and publish IRs. t := &gatewayapi.Translator{ ``` Here's the o/p and its empty ``` 2023-07-27T23:55:29.795Z INFO gateway-api runner/runner.go:62 {"runner": "gateway-api", "output": "diff"} ``` * Using a DeepCopy for resources that were updating the `Status` subresource seems to have solved the issue, which implies that watchable doesnt like clients to mutate the value, even though they are meant to be a `DeepCopy` Fixes: https://github.com/envoyproxy/gateway/issues/1715 Signed-off-by: Arko Dasgupta (cherry picked from commit 5b72451b8eec7362f1c204c0955daa022f1df37a) * observability: add container port for metrics (#1736) container port Signed-off-by: zirain (cherry picked from commit 4bba03abd5af7bd85450b37c9e4b331655ade6c4) * docs: Add user docs for EnvoyPatchPolicy (#1733) * Add user docs for EnvoyPatchPolicy Relates to https://github.com/envoyproxy/gateway/issues/24 Signed-off-by: Arko Dasgupta * nits Signed-off-by: Arko Dasgupta * wrap up Signed-off-by: Arko Dasgupta * lint Signed-off-by: Arko Dasgupta * address comments && fix config Signed-off-by: Arko Dasgupta --------- Signed-off-by: Arko Dasgupta (cherry picked from commit 27b0939248801c673046c11a6669695998272ede) * e2e & misc fixes for EnvoyPatchPolicy (#1738) * Add E2E for EnvoyPatchPolicy * Use LocalReplyConfig to return a custom status code `406` when there is no valid route match Signed-off-by: Arko Dasgupta (cherry picked from commit a7784c5be3a949e9ac01577598b5c4148c2a8ae8) --------- Signed-off-by: Arko Dasgupta Co-authored-by: qi Co-authored-by: zirain --- api/v1alpha1/envoypatchpolicy_types.go | 2 +- ...eway.envoyproxy.io_envoypatchpolicies.yaml | 1 - docs/latest/user/envoy-patch-policy.md | 200 ++++++++++++++++++ docs/latest/user_docs.rst | 1 + examples/kubernetes/metric/pod-monitor.yaml | 17 ++ examples/redis/redis.yaml | 2 + internal/gatewayapi/envoypatchpolicy.go | 2 +- internal/gatewayapi/route.go | 10 +- internal/gatewayapi/runner/runner.go | 2 + .../kubernetes/proxy/resource.go | 22 +- .../kubernetes/proxy/resource_provider.go | 2 + .../proxy/resource_provider_test.go | 3 +- .../proxy/testdata/deployments/bootstrap.yaml | 8 + .../testdata/deployments/component-level.yaml | 8 + .../proxy/testdata/deployments/custom.yaml | 8 + .../testdata/deployments/default-env.yaml | 8 + .../proxy/testdata/deployments/default.yaml | 8 + .../deployments/enable-prometheus.yaml | 11 + .../testdata/deployments/extension-env.yaml | 8 + .../proxy/testdata/deployments/volumes.yaml | 8 + .../kubernetes/proxy_infra_test.go | 13 ++ .../kubernetes/ratelimit/resource.go | 7 +- .../kubernetes/ratelimit/resource_provider.go | 2 + .../ratelimit/resource_provider_test.go | 3 +- .../testdata/deployments/affinity.yaml | 3 + .../testdata/deployments/custom.yaml | 3 + .../testdata/deployments/default-env.yaml | 3 + .../testdata/deployments/default.yaml | 3 + .../testdata/deployments/extension-env.yaml | 3 + .../testdata/deployments/override-env.yaml | 3 + .../deployments/redis-tls-settings.yaml | 4 + .../testdata/deployments/tolerations.yaml | 4 + .../testdata/deployments/volumes.yaml | 4 + .../kubernetes/resource/resource_test.go | 5 +- internal/infrastructure/runner/runner.go | 1 + internal/provider/kubernetes/controller.go | 23 +- internal/status/envoypatchpolicy.go | 3 + internal/xds/server/runner/runner.go | 1 + test/e2e/testdata/envoy-patch-policy.yaml | 48 +++++ test/e2e/tests/envoy-patch-policy.go | 61 ++++++ 40 files changed, 499 insertions(+), 29 deletions(-) create mode 100644 docs/latest/user/envoy-patch-policy.md create mode 100644 examples/kubernetes/metric/pod-monitor.yaml create mode 100644 test/e2e/testdata/envoy-patch-policy.yaml create mode 100644 test/e2e/tests/envoy-patch-policy.go diff --git a/api/v1alpha1/envoypatchpolicy_types.go b/api/v1alpha1/envoypatchpolicy_types.go index 8234ead224e..012ac42a989 100644 --- a/api/v1alpha1/envoypatchpolicy_types.go +++ b/api/v1alpha1/envoypatchpolicy_types.go @@ -59,7 +59,7 @@ type EnvoyPatchPolicySpec struct { // the priority i.e. int32.min has the highest priority and // int32.max has the lowest priority. // Defaults to 0. - Priority int32 `json:"priority"` + Priority int32 `json:"priority,omitempty"` } // EnvoyPatchType specifies the types of Envoy patching mechanisms. diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoypatchpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoypatchpolicies.yaml index c336131fcca..7296a9e5127 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoypatchpolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoypatchpolicies.yaml @@ -142,7 +142,6 @@ spec: - JSONPatch type: string required: - - priority - targetRef - type type: object diff --git a/docs/latest/user/envoy-patch-policy.md b/docs/latest/user/envoy-patch-policy.md new file mode 100644 index 00000000000..fc0f63fb31c --- /dev/null +++ b/docs/latest/user/envoy-patch-policy.md @@ -0,0 +1,200 @@ +# Envoy Patch Policy + +This guide explains the usage of the [EnvoyPatchPolicy][] API. +__Note:__ This API is meant for users extremely familiar with Envoy [xDS][] semantics. +Also before considering this API for production use cases, please be aware that this API +is unstable and the outcome may change across versions. Use at your own risk. + +## Introduction + +The [EnvoyPatchPolicy][] API allows user to modify the output [xDS][] +configuration generated by Envoy Gateway intended for EnvoyProxy, +using [JSON Patch][] semantics. + +## Motivation + +This API was introduced to allow advanced users to be able to leverage Envoy Proxy functionality +not exposed by Envoy Gateway APIs today. + +## Quickstart + +### Prerequistes + +* Follow the steps from the [Quickstart](quickstart.md) guide to install Envoy Gateway and the example manifest. +Before proceeding, you should be able to query the example backend using HTTP. + +### Enable EnvoyPatchPolicy + +* By default EnvoyPatchPolicy][] is disabled. Lets enable it in the [EnvoyGateway][] startup configuration + +* The default installation of Envoy Gateway installs a default [EnvoyGateway][] configuration and attaches it +using a `ConfigMap`. In the next step, we will update this resource to enable EnvoyPatchPolicy. + + +```shell +cat <// + name: default/eg/http + operation: + op: add + path: "/default_filter_chain/filters/0/typed_config/local_reply_config" + value: + mappers: + - filter: + status_code_filter: + comparison: + op: EQ + value: + default_value: 404 + runtime_key: key_b + status_code: 406 + body: + inline_string: "could not find what you are looking for" +EOF +``` + +* Lets edit the HTTPRoute resource from the Quickstart to only match on paths with value `/get` + +``` +kubectl patch httproute backend --type=json --patch '[{ + "op": "add", + "path": "/spec/rules/0/matches/0/path/value", + "value": "/get", +}]' +``` + +* Lets test it out by specifying a path apart from `/get` + +``` +$ curl --header "Host: www.example.com" http://localhost:8888/find +Handling connection for 8888 +could not find what you are looking for +``` + +## Debugging + +### Runtime + +* The `Status` subresource should have information about the status of the resource. Make sure +`Accepted=True` and `Programmed=True` conditions are set to ensure that the policy has been +applied to Envoy Proxy. + +``` +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"gateway.envoyproxy.io/v1alpha1","kind":"EnvoyPatchPolicy","metadata":{"annotations":{},"name":"custom-response-patch-policy","namespace":"default"},"spec":{"jsonPatches":[{"name":"default/eg/http","operation":{"op":"add","path":"/default_filter_chain/filters/0/typed_config/local_reply_config","value":{"mappers":[{"body":{"inline_string":"could not find what you are looking for"},"filter":{"status_code_filter":{"comparison":{"op":"EQ","value":{"default_value":404}}}}}]}},"type":"type.googleapis.com/envoy.config.listener.v3.Listener"}],"priority":0,"targetRef":{"group":"gateway.networking.k8s.io","kind":"Gateway","name":"eg","namespace":"default"},"type":"JSONPatch"}} + creationTimestamp: "2023-07-31T21:47:53Z" + generation: 1 + name: custom-response-patch-policy + namespace: default + resourceVersion: "10265" + uid: a35bda6e-a0cc-46d7-a63a-cee765174bc3 +spec: + jsonPatches: + - name: default/eg/http + operation: + op: add + path: /default_filter_chain/filters/0/typed_config/local_reply_config + value: + mappers: + - body: + inline_string: could not find what you are looking for + filter: + status_code_filter: + comparison: + op: EQ + value: + default_value: 404 + type: type.googleapis.com/envoy.config.listener.v3.Listener + priority: 0 + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: eg + namespace: default + type: JSONPatch +status: + conditions: + - lastTransitionTime: "2023-07-31T21:48:19Z" + message: EnvoyPatchPolicy has been accepted. + observedGeneration: 1 + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: "2023-07-31T21:48:19Z" + message: successfully applied patches. + reason: Programmed + status: "True" + type: Programmed +``` + +### Offline + +* You can use [egctl x translate][] to validate the translated xds output. + +## Caveats + +This API will always be an unstable API and the same outcome cannot be garunteed +across versions for these reasons +* The Envoy Proxy API might deprecate and remove API fields +* Envoy Gateway might alter the xDS translation creating a different xDS output +such as changing the `name` field of resources. + +[EnvoyPatchPolicy]: https://gateway.envoyproxy.io/latest/api/extension_types.html#envoypatchpolicy +[EnvoyGateway]: https://gateway.envoyproxy.io/latest/api/config_types.html#envoygateway +[JSON Patch]: https://datatracker.ietf.org/doc/html/rfc6902 +[xDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration +[Local Reply Modification]: https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/local_reply +[egctl x translate]: https://gateway.envoyproxy.io/latest/user/egctl.html#egctl-experimental-translate diff --git a/docs/latest/user_docs.rst b/docs/latest/user_docs.rst index faa4875c4e6..eb0fb7714ba 100644 --- a/docs/latest/user_docs.rst +++ b/docs/latest/user_docs.rst @@ -23,6 +23,7 @@ Learn how to deploy, use, and operate Envoy Gateway. user/grpc-routing user/authn user/rate-limit + user/envoy-patch-policy user/egctl user/customize-envoyproxy user/deployment-mode diff --git a/examples/kubernetes/metric/pod-monitor.yaml b/examples/kubernetes/metric/pod-monitor.yaml new file mode 100644 index 00000000000..dc41549881b --- /dev/null +++ b/examples/kubernetes/metric/pod-monitor.yaml @@ -0,0 +1,17 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: envoy-gateway-proxy-monitoring + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: envoy + app.kubernetes.io/component: proxy + namespaceSelector: + any: true + jobLabel: proxy-stats + podMetricsEndpoints: + - path: /stats/prometheus + interval: 15s + port: metrics diff --git a/examples/redis/redis.yaml b/examples/redis/redis.yaml index c0b0992199f..74f6f6788de 100644 --- a/examples/redis/redis.yaml +++ b/examples/redis/redis.yaml @@ -62,6 +62,8 @@ data: type: Kubernetes gateway: controllerName: gateway.envoyproxy.io/gatewayclass-controller + extensionApis: + enableEnvoyPatchPolicy: true rateLimit: backend: type: Redis diff --git a/internal/gatewayapi/envoypatchpolicy.go b/internal/gatewayapi/envoypatchpolicy.go index 4a9ff8cc6e4..e5e9a2f33e0 100644 --- a/internal/gatewayapi/envoypatchpolicy.go +++ b/internal/gatewayapi/envoypatchpolicy.go @@ -25,7 +25,7 @@ func (t *Translator) ProcessEnvoyPatchPolicies(envoyPatchPolicies []*egv1a1.Envo }) for _, policy := range envoyPatchPolicies { - + policy := policy.DeepCopy() targetNs := policy.Spec.TargetRef.Namespace if targetNs == nil { // This status condition will not get updated in the resource because diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index dee403e2ef7..8fc96264b95 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -39,7 +39,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } httpRoute := &HTTPRouteContext{ GatewayControllerName: t.GatewayControllerName, - HTTPRoute: h, + HTTPRoute: h.DeepCopy(), } // Find out if this route attaches to one of our Gateway's listeners, @@ -67,7 +67,7 @@ func (t *Translator) ProcessGRPCRoutes(grpcRoutes []*v1alpha2.GRPCRoute, gateway } grpcRoute := &GRPCRouteContext{ GatewayControllerName: t.GatewayControllerName, - GRPCRoute: g, + GRPCRoute: g.DeepCopy(), } // Find out if this route attaches to one of our Gateway's listeners, @@ -563,7 +563,7 @@ func (t *Translator) ProcessTLSRoutes(tlsRoutes []*v1alpha2.TLSRoute, gateways [ } tlsRoute := &TLSRouteContext{ GatewayControllerName: t.GatewayControllerName, - TLSRoute: tls, + TLSRoute: tls.DeepCopy(), } // Find out if this route attaches to one of our Gateway's listeners, @@ -685,7 +685,7 @@ func (t *Translator) ProcessUDPRoutes(udpRoutes []*v1alpha2.UDPRoute, gateways [ } udpRoute := &UDPRouteContext{ GatewayControllerName: t.GatewayControllerName, - UDPRoute: u, + UDPRoute: u.DeepCopy(), } // Find out if this route attaches to one of our Gateway's listeners, @@ -817,7 +817,7 @@ func (t *Translator) ProcessTCPRoutes(tcpRoutes []*v1alpha2.TCPRoute, gateways [ } tcpRoute := &TCPRouteContext{ GatewayControllerName: t.GatewayControllerName, - TCPRoute: tcp, + TCPRoute: tcp.DeepCopy(), } // Find out if this route attaches to one of our Gateway's listeners, diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index 050394ba308..7db9c05b962 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -51,6 +51,8 @@ func (r *Runner) Start(ctx context.Context) error { func (r *Runner) subscribeAndTranslate(ctx context.Context) { message.HandleSubscription(r.ProviderResources.GatewayAPIResources.Subscribe(ctx), func(update message.Update[string, *gatewayapi.Resources]) { + r.Logger.Info("received an update") + val := update.Value if update.Delete || val == nil { diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 70c2a977ddf..119dc877441 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -117,6 +117,14 @@ func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egcfgv1a1.K proxyMetrics = infra.Config.Spec.Telemetry.Metrics } + if proxyMetrics != nil && proxyMetrics.Prometheus != nil { + ports = append(ports, corev1.ContainerPort{ + Name: "metrics", + ContainerPort: bootstrap.EnvoyReadinessPort, // TODO: make this configurable + Protocol: corev1.ProtocolTCP, + }) + } + var bootstrapConfigurations string // Get Bootstrap from EnvoyProxy API if set by the user // The config should have been validated already @@ -163,10 +171,15 @@ func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egcfgv1a1.K ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: bootstrap.EnvoyReadinessPath, - Port: intstr.IntOrString{Type: intstr.Int, IntVal: bootstrap.EnvoyReadinessPort}, + Path: bootstrap.EnvoyReadinessPath, + Port: intstr.IntOrString{Type: intstr.Int, IntVal: bootstrap.EnvoyReadinessPort}, + Scheme: corev1.URISchemeHTTP, }, }, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, }, } @@ -222,7 +235,8 @@ func expectedDeploymentVolumes(name string, deploymentSpec *egcfgv1a1.Kubernetes Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "envoy", + SecretName: "envoy", + DefaultMode: pointer.Int32(420), }, }, }, @@ -243,7 +257,7 @@ func expectedDeploymentVolumes(name string, deploymentSpec *egcfgv1a1.Kubernetes Path: SdsCertFilename, }, }, - DefaultMode: pointer.Int32(int32(420)), + DefaultMode: pointer.Int32(420), Optional: pointer.Bool(false), }, }, diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index 7f1b4aa9897..186c91c9f12 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -224,6 +224,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Volumes: expectedDeploymentVolumes(r.infra.Name, deploymentConfig), }, }, + RevisionHistoryLimit: pointer.Int32(10), + ProgressDeadlineSeconds: pointer.Int32(600), }, } diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 043d54b9bf6..2f08f795b55 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -208,7 +208,8 @@ func TestDeployment(t *testing.T) { Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "custom-envoy-cert", + SecretName: "custom-envoy-cert", + DefaultMode: pointer.Int32(420), }, }, }, diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml index eddd6d37cc2..1e736c818f0 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml @@ -67,6 +67,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -84,6 +89,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -94,3 +100,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml index a765fbbb6a2..35502631411 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml @@ -68,6 +68,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -85,6 +90,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -95,3 +101,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml index 583aba05b1f..7b24c5fecb9 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml @@ -171,6 +171,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: @@ -192,6 +197,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -202,3 +208,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml index b67ef66112b..9535bc2e1e4 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml @@ -169,6 +169,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: @@ -190,6 +195,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -200,3 +206,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml index 8e4a36f6e82..e2fd52c8384 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml @@ -164,6 +164,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -181,6 +186,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -191,3 +197,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml index d9397a4e8c6..b6c181a3709 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml @@ -182,6 +182,9 @@ spec: - containerPort: 8443 name: EnvoyHTTPSPort protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP resources: requests: cpu: 100m @@ -190,6 +193,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -207,6 +215,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -217,3 +226,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml index 7d37f7ebe57..b0b2a4640a7 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml @@ -173,6 +173,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: @@ -194,6 +199,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -204,3 +210,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml index 31010e9f142..b26628f1e27 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml @@ -173,6 +173,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: @@ -194,6 +199,7 @@ spec: - name: certs secret: secretName: custom-envoy-cert + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -204,3 +210,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy_infra_test.go b/internal/infrastructure/kubernetes/proxy_infra_test.go index 7328b107064..d3180439d69 100644 --- a/internal/infrastructure/kubernetes/proxy_infra_test.go +++ b/internal/infrastructure/kubernetes/proxy_infra_test.go @@ -7,8 +7,10 @@ package kubernetes import ( "context" + "reflect" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -30,6 +32,17 @@ func newTestInfra(t *testing.T) *Infra { return newTestInfraWithClient(t, cli) } +func TestCmpBytes(t *testing.T) { + m1 := map[string][]byte{} + m1["a"] = []byte("aaa") + m2 := map[string][]byte{} + m2["a"] = []byte("aaa") + + assert.True(t, reflect.DeepEqual(m1, m2)) + assert.False(t, reflect.DeepEqual(nil, m2)) + assert.False(t, reflect.DeepEqual(m1, nil)) +} + func newTestInfraWithClient(t *testing.T, cli client.Client) *Infra { cfg, err := config.New() require.NoError(t, err) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource.go b/internal/infrastructure/kubernetes/ratelimit/resource.go index f35de689d4a..e7a39f8383c 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource.go @@ -12,6 +12,7 @@ import ( "strconv" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" @@ -161,7 +162,8 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme Name: "redis-certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: string(rateLimit.Backend.Redis.TLS.CertificateRef.Name), + SecretName: string(rateLimit.Backend.Redis.TLS.CertificateRef.Name), + DefaultMode: pointer.Int32(420), }, }, }) @@ -171,7 +173,8 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "envoy-rate-limit", + SecretName: "envoy-rate-limit", + DefaultMode: pointer.Int32(420), }, }, }) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 8bc7a2ffc69..ba7505fc065 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -180,6 +180,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Tolerations: r.rateLimitDeployment.Pod.Tolerations, }, }, + RevisionHistoryLimit: pointer.Int32(10), + ProgressDeadlineSeconds: pointer.Int32(600), }, } diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index adf7750129a..81580b1d205 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -444,7 +444,8 @@ func TestDeployment(t *testing.T) { Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "custom-cert", + SecretName: "custom-cert", + DefaultMode: pointer.Int32(420), }, }, }, diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml index c28e2444be6..ba83d0237dc 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml @@ -115,3 +115,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml index a0bb2e4ea1e..d3c0f29dc48 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml @@ -106,3 +106,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml index a0bb2e4ea1e..d3c0f29dc48 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml @@ -106,3 +106,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml index 90e6a1a1bf6..c95a4fc420b 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml @@ -97,3 +97,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml index bbeea5e2170..4b6457ebdda 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml @@ -110,3 +110,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml index b5b21bd4692..6371fea87a9 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml @@ -106,3 +106,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml index 870b647c355..dacf0988f59 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml @@ -117,6 +117,10 @@ spec: - name: redis-certs secret: secretName: ratelimit-cert + defaultMode: 420 - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml index 17f712c6695..23b2791b8e0 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml @@ -122,6 +122,10 @@ spec: - name: redis-certs secret: secretName: ratelimit-cert + defaultMode: 420 - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml index d5c8eb5d2a2..0fba367bdb8 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml @@ -122,6 +122,10 @@ spec: - name: redis-certs secret: secretName: ratelimit-cert-origin + defaultMode: 420 - name: certs secret: secretName: custom-cert + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/resource/resource_test.go b/internal/infrastructure/kubernetes/resource/resource_test.go index 314fa4e650e..46730960851 100644 --- a/internal/infrastructure/kubernetes/resource/resource_test.go +++ b/internal/infrastructure/kubernetes/resource/resource_test.go @@ -8,12 +8,11 @@ package resource import ( "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" ) diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index 03bead88ef6..2f7bd146df9 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -57,6 +57,7 @@ func (r *Runner) subscribeToProxyInfraIR(ctx context.Context) { // Subscribe to resources message.HandleSubscription(r.InfraIR.Subscribe(ctx), func(update message.Update[string, *ir.Infra]) { + r.Logger.Info("received an update") val := update.Value if update.Delete { diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 25255cd0c45..1e26fd0bde3 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -240,16 +240,19 @@ func (r *gatewayAPIReconciler) Reconcile(ctx context.Context, request reconcile. } // Add all EnvoyPatchPolicies - envoyPatchPolicies := egv1a1.EnvoyPatchPolicyList{} - if err := r.client.List(ctx, &envoyPatchPolicies); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing envoypatchpolicies: %v", err) - } - for _, policy := range envoyPatchPolicies.Items { - policy := policy - // Discard Status to reduce memory consumption in watchable - // It will be recomputed by the gateway-api layer - policy.Status = egv1a1.EnvoyPatchPolicyStatus{} - resourceTree.EnvoyPatchPolicies = append(resourceTree.EnvoyPatchPolicies, &policy) + if r.envoyGateway.ExtensionAPIs != nil && r.envoyGateway.ExtensionAPIs.EnableEnvoyPatchPolicy { + envoyPatchPolicies := egv1a1.EnvoyPatchPolicyList{} + if err := r.client.List(ctx, &envoyPatchPolicies); err != nil { + return reconcile.Result{}, fmt.Errorf("error listing envoypatchpolicies: %v", err) + } + + for _, policy := range envoyPatchPolicies.Items { + policy := policy + // Discard Status to reduce memory consumption in watchable + // It will be recomputed by the gateway-api layer + policy.Status = egv1a1.EnvoyPatchPolicyStatus{} + resourceTree.EnvoyPatchPolicies = append(resourceTree.EnvoyPatchPolicies, &policy) + } } // For this particular Gateway, and all associated objects, check whether the diff --git a/internal/status/envoypatchpolicy.go b/internal/status/envoypatchpolicy.go index 950ce1f58ae..05deb89d154 100644 --- a/internal/status/envoypatchpolicy.go +++ b/internal/status/envoypatchpolicy.go @@ -25,6 +25,9 @@ func SetEnvoyPatchPolicyProgrammedIfUnset(s *egv1a1.EnvoyPatchPolicyStatus, mess if c.Type == string(egv1a1.PolicyConditionProgrammed) { return } + if c.Type == string(gwv1a2.PolicyConditionAccepted) && c.Status == metav1.ConditionFalse { + return + } } cond := newCondition(string(egv1a1.PolicyConditionProgrammed), metav1.ConditionTrue, string(egv1a1.PolicyReasonProgrammed), message, time.Now(), 0) diff --git a/internal/xds/server/runner/runner.go b/internal/xds/server/runner/runner.go index c3482355a55..e76224a739c 100644 --- a/internal/xds/server/runner/runner.go +++ b/internal/xds/server/runner/runner.go @@ -130,6 +130,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { key := update.Key val := update.Value + r.Logger.Info("received an update") var err error if update.Delete { err = r.cache.GenerateNewSnapshot(key, nil) diff --git a/test/e2e/testdata/envoy-patch-policy.yaml b/test/e2e/testdata/envoy-patch-policy.yaml new file mode 100644 index 00000000000..fa63ead7c80 --- /dev/null +++ b/test/e2e/testdata/envoy-patch-policy.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: http-envoy-patch-policy + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - backendRefs: + - name: infra-backend-v1 + port: 8080 + matches: + - path: + type: PathPrefix + value: /foo +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: custom-response-patch-policy + namespace: gateway-conformance-infra +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: same-namespace + namespace: gateway-conformance-infra + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.listener.v3.Listener" + name: "gateway-conformance-infra/same-namespace/http" + operation: + op: add + path: "/default_filter_chain/filters/0/typed_config/local_reply_config" + value: + mappers: + - filter: + status_code_filter: + comparison: + op: EQ + value: + default_value: 404 + runtime_key: key_b + status_code: 406 + body: + inline_string: "not acceptable" diff --git a/test/e2e/tests/envoy-patch-policy.go b/test/e2e/tests/envoy-patch-policy.go new file mode 100644 index 00000000000..2486518ac31 --- /dev/null +++ b/test/e2e/tests/envoy-patch-policy.go @@ -0,0 +1,61 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e +// +build e2e + +package tests + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, EnvoyPatchPolicyTest) +} + +var EnvoyPatchPolicyTest = suite.ConformanceTest{ + ShortName: "EnvoyPatchPolicy", + Description: "update xds using EnvoyPatchPolicy", + Manifests: []string{"testdata/envoy-patch-policy.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + t.Run("envoy patch policy", func(t *testing.T) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "http-envoy-patch-policy", Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + OkResp := http.ExpectedResponse{ + Request: http.Request{ + Path: "/foo", + }, + Response: http.Response{ + StatusCode: 200, + }, + Namespace: ns, + } + + // Send a request to an valid path and expect a successful response + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, OkResp) + + customResp := http.ExpectedResponse{ + Request: http.Request{ + Path: "/bar", + }, + Response: http.Response{ + StatusCode: 406, + }, + Namespace: ns, + } + + // Send a request to an invalid path and expect a custom response + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, customResp) + }) + }, +}