From fcdb9b168c823d2fd47dd94c3edfc4c5db984f42 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Mon, 10 Oct 2022 23:06:40 +0800 Subject: [PATCH 001/113] fix(conformance): add HTTPRouteHeaderMatching case (#532) Signed-off-by: bitliu --- test/conformance/conformance_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index e69f5586b71..6273fd69e1c 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -50,6 +50,7 @@ func TestGatewayAPIConformance(t *testing.T) { tests.HTTPRouteInvalidNonExistentBackendRef, tests.HTTPRouteInvalidBackendRefUnknownKind, tests.HTTPRouteInvalidCrossNamespaceBackendRef, + tests.HTTPRouteHeaderMatching, } cSuite.Run(t, egTests) From 34afca0f0ad17e4caed3cda7a1ac2a1a1658ab11 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Mon, 10 Oct 2022 23:44:58 +0800 Subject: [PATCH 002/113] fix: cobra mismatched comments (#525) Signed-off-by: bitliu --- internal/cmd/certgen.go | 2 +- internal/cmd/versions.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cmd/certgen.go b/internal/cmd/certgen.go index 5d7a012106a..70a425a12cb 100644 --- a/internal/cmd/certgen.go +++ b/internal/cmd/certgen.go @@ -16,7 +16,7 @@ import ( "github.com/envoyproxy/gateway/internal/provider/kubernetes" ) -// getServerCommand returns the server cobra command to be executed. +// getCertGenCommand returns the certGen cobra command to be executed. func getCertGenCommand() *cobra.Command { cmd := &cobra.Command{ Use: "certgen", diff --git a/internal/cmd/versions.go b/internal/cmd/versions.go index 7b15712647a..2439be8836e 100644 --- a/internal/cmd/versions.go +++ b/internal/cmd/versions.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -// getVersionsCommand returns the server cobra command to be executed. +// getVersionsCommand returns the version cobra command to be executed. func getVersionsCommand() *cobra.Command { // envOutput determines whether to output as environment settings var envOutput bool From ce1aa350905369fa2588ef3e13789fe6d5c730e1 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 11 Oct 2022 02:01:57 +0800 Subject: [PATCH 003/113] fix: set correct listenerContext order (#535) fix: set correct listener context order Signed-off-by: bitliu --- internal/gatewayapi/contexts.go | 12 +- internal/gatewayapi/helpers.go | 4 +- ...eway-with-more-different-listeners.in.yaml | 82 +++++ ...way-with-more-different-listeners.out.yaml | 326 ++++++++++++++++++ ...ing-to-gateway-with-more-listeners.in.yaml | 82 +++++ ...ng-to-gateway-with-more-listeners.out.yaml | 298 ++++++++++++++++ 6 files changed, 797 insertions(+), 7 deletions(-) create mode 100644 internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml create mode 100644 internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml diff --git a/internal/gatewayapi/contexts.go b/internal/gatewayapi/contexts.go index 23523401c70..3b1aec98c56 100644 --- a/internal/gatewayapi/contexts.go +++ b/internal/gatewayapi/contexts.go @@ -17,16 +17,18 @@ import ( type GatewayContext struct { *v1beta1.Gateway - listeners map[v1beta1.SectionName]*ListenerContext + listeners []*ListenerContext } func (g *GatewayContext) GetListenerContext(listenerName v1beta1.SectionName) *ListenerContext { if g.listeners == nil { - g.listeners = make(map[v1beta1.SectionName]*ListenerContext) + g.listeners = make([]*ListenerContext, 0) } - if ctx := g.listeners[listenerName]; ctx != nil { - return ctx + for _, l := range g.listeners { + if l.Name == listenerName { + return l + } } var listener *v1beta1.Listener @@ -57,7 +59,7 @@ func (g *GatewayContext) GetListenerContext(listenerName v1beta1.SectionName) *L gateway: g.Gateway, listenerStatusIdx: listenerStatusIdx, } - g.listeners[listenerName] = ctx + g.listeners = append(g.listeners, ctx) return ctx } diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index f4313c93b29..4b54b610716 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -117,8 +117,8 @@ func GetReferencedListeners(parentRef v1beta1.ParentReference, gateways []*Gatew selectsGateway = true // The parentRef may be to the entire Gateway, or to a specific listener. - for listenerName, listener := range gateway.listeners { - if parentRef.SectionName == nil || *parentRef.SectionName == listenerName { + for _, listener := range gateway.listeners { + if parentRef.SectionName == nil || *parentRef.SectionName == listener.Name { referencedListeners = append(referencedListeners, listener) } } diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.in.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.in.yaml new file mode 100644 index 00000000000..7ec5ee38330 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.in.yaml @@ -0,0 +1,82 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http-1 + protocol: HTTP + port: 81 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + - name: http-2 + protocol: HTTP + port: 82 + hostname: bar.com + allowedRoutes: + namespaces: + from: All + - name: http-3 + protocol: HTTP + port: 83 + hostname: foo1.com + allowedRoutes: + namespaces: + from: All + - name: http-4 + protocol: HTTP + port: 84 + hostname: bar1.com + allowedRoutes: + namespaces: + from: All + - name: http-5 + protocol: HTTP + port: 85 + hostname: foo2.com + allowedRoutes: + namespaces: + from: All + - name: http-6 + protocol: HTTP + port: 86 + hostname: bar2.com + allowedRoutes: + namespaces: + from: All + - name: http-7 + protocol: HTTP + port: 87 + hostname: foo3.com + allowedRoutes: + namespaces: + from: All + - name: http-8 + protocol: HTTP + port: 88 + hostname: bar3.com + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml new file mode 100644 index 00000000000..c93b03ed7df --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml @@ -0,0 +1,326 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http-1 + protocol: HTTP + port: 81 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + - name: http-2 + protocol: HTTP + port: 82 + hostname: bar.com + allowedRoutes: + namespaces: + from: All + - name: http-3 + protocol: HTTP + port: 83 + hostname: foo1.com + allowedRoutes: + namespaces: + from: All + - name: http-4 + protocol: HTTP + port: 84 + hostname: bar1.com + allowedRoutes: + namespaces: + from: All + - name: http-5 + protocol: HTTP + port: 85 + hostname: foo2.com + allowedRoutes: + namespaces: + from: All + - name: http-6 + protocol: HTTP + port: 86 + hostname: bar2.com + allowedRoutes: + namespaces: + from: All + - name: http-7 + protocol: HTTP + port: 87 + hostname: foo3.com + allowedRoutes: + namespaces: + from: All + - name: http-8 + protocol: HTTP + port: 88 + hostname: bar3.com + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: http-1 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-2 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-3 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-4 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-5 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-6 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-7 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-8 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + http: + - name: envoy-gateway-gateway-1-http-1 + address: 0.0.0.0 + port: 10081 + hostnames: + - foo.com + routes: + - name: default-httproute-1-rule-0-match-0-foo.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-2 + address: 0.0.0.0 + port: 10082 + hostnames: + - bar.com + routes: + - name: default-httproute-1-rule-0-match-0-bar.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-3 + address: 0.0.0.0 + port: 10083 + hostnames: + - foo1.com + routes: + - name: default-httproute-1-rule-0-match-0-foo1.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-4 + address: 0.0.0.0 + port: 10084 + hostnames: + - bar1.com + routes: + - name: default-httproute-1-rule-0-match-0-bar1.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-5 + address: 0.0.0.0 + port: 10085 + hostnames: + - foo2.com + routes: + - name: default-httproute-1-rule-0-match-0-foo2.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-6 + address: 0.0.0.0 + port: 10086 + hostnames: + - bar2.com + routes: + - name: default-httproute-1-rule-0-match-0-bar2.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-7 + address: 0.0.0.0 + port: 10087 + hostnames: + - foo3.com + routes: + - name: default-httproute-1-rule-0-match-0-foo3.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-8 + address: 0.0.0.0 + port: 10088 + hostnames: + - bar3.com + routes: + - name: default-httproute-1-rule-0-match-0-bar3.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" + ports: + - name: http-1 + protocol: "HTTP" + servicePort: 81 + containerPort: 10081 + - name: http-2 + protocol: "HTTP" + servicePort: 82 + containerPort: 10082 + - name: http-3 + protocol: "HTTP" + servicePort: 83 + containerPort: 10083 + - name: http-4 + protocol: "HTTP" + servicePort: 84 + containerPort: 10084 + - name: http-5 + protocol: "HTTP" + servicePort: 85 + containerPort: 10085 + - name: http-6 + protocol: "HTTP" + servicePort: 86 + containerPort: 10086 + - name: http-7 + protocol: "HTTP" + servicePort: 87 + containerPort: 10087 + - name: http-8 + protocol: "HTTP" + servicePort: 88 + containerPort: 10088 diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.in.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.in.yaml new file mode 100644 index 00000000000..6d6d34395df --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.in.yaml @@ -0,0 +1,82 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http-1 + protocol: HTTP + port: 80 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + - name: http-2 + protocol: HTTP + port: 80 + hostname: bar.com + allowedRoutes: + namespaces: + from: All + - name: http-3 + protocol: HTTP + port: 80 + hostname: foo1.com + allowedRoutes: + namespaces: + from: All + - name: http-4 + protocol: HTTP + port: 80 + hostname: bar1.com + allowedRoutes: + namespaces: + from: All + - name: http-5 + protocol: HTTP + port: 80 + hostname: foo2.com + allowedRoutes: + namespaces: + from: All + - name: http-6 + protocol: HTTP + port: 80 + hostname: bar2.com + allowedRoutes: + namespaces: + from: All + - name: http-7 + protocol: HTTP + port: 80 + hostname: foo3.com + allowedRoutes: + namespaces: + from: All + - name: http-8 + protocol: HTTP + port: 80 + hostname: bar3.com + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml new file mode 100644 index 00000000000..b0c96ad796f --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml @@ -0,0 +1,298 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http-1 + protocol: HTTP + port: 80 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + - name: http-2 + protocol: HTTP + port: 80 + hostname: bar.com + allowedRoutes: + namespaces: + from: All + - name: http-3 + protocol: HTTP + port: 80 + hostname: foo1.com + allowedRoutes: + namespaces: + from: All + - name: http-4 + protocol: HTTP + port: 80 + hostname: bar1.com + allowedRoutes: + namespaces: + from: All + - name: http-5 + protocol: HTTP + port: 80 + hostname: foo2.com + allowedRoutes: + namespaces: + from: All + - name: http-6 + protocol: HTTP + port: 80 + hostname: bar2.com + allowedRoutes: + namespaces: + from: All + - name: http-7 + protocol: HTTP + port: 80 + hostname: foo3.com + allowedRoutes: + namespaces: + from: All + - name: http-8 + protocol: HTTP + port: 80 + hostname: bar3.com + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: http-1 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-2 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-3 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-4 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-5 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-6 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-7 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: http-8 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + http: + - name: envoy-gateway-gateway-1-http-1 + address: 0.0.0.0 + port: 10080 + hostnames: + - foo.com + routes: + - name: default-httproute-1-rule-0-match-0-foo.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-2 + address: 0.0.0.0 + port: 10080 + hostnames: + - bar.com + routes: + - name: default-httproute-1-rule-0-match-0-bar.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-3 + address: 0.0.0.0 + port: 10080 + hostnames: + - foo1.com + routes: + - name: default-httproute-1-rule-0-match-0-foo1.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-4 + address: 0.0.0.0 + port: 10080 + hostnames: + - bar1.com + routes: + - name: default-httproute-1-rule-0-match-0-bar1.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-5 + address: 0.0.0.0 + port: 10080 + hostnames: + - foo2.com + routes: + - name: default-httproute-1-rule-0-match-0-foo2.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-6 + address: 0.0.0.0 + port: 10080 + hostnames: + - bar2.com + routes: + - name: default-httproute-1-rule-0-match-0-bar2.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-7 + address: 0.0.0.0 + port: 10080 + hostnames: + - foo3.com + routes: + - name: default-httproute-1-rule-0-match-0-foo3.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-http-8 + address: 0.0.0.0 + port: 10080 + hostnames: + - bar3.com + routes: + - name: default-httproute-1-rule-0-match-0-bar3.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" + ports: + - name: http-1 + protocol: "HTTP" + servicePort: 80 + containerPort: 10080 From 81eb8178550e7dfe111caf329390e5554c61d0c8 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 11 Oct 2022 02:17:21 +0800 Subject: [PATCH 004/113] fix: make kube-undeploy target failed (#528) Signed-off-by: bitliu --- tools/make/kube.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/make/kube.mk b/tools/make/kube.mk index c879c88c3c5..f458e7f7d55 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -31,7 +31,7 @@ kube-test: manifests generate $(tools/setup-envtest) ## Run Kubernetes provider ##@ Kubernetes Deployment ifndef ignore-not-found - ignore-not-found = false + ignore-not-found = true endif .PHONY: kube-install @@ -45,7 +45,7 @@ kube-install: manifests $(tools/kustomize) ## Install Envoy Gateway CRDs into th .PHONY: kube-uninstall kube-uninstall: manifests $(tools/kustomize) ## Uninstall Envoy Gateway CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml + kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml --ignore-not-found=$(ignore-not-found) .PHONY: kube-deploy kube-deploy: kube-install ## Install Envoy Gateway controller into the Kubernetes cluster specified in ~/.kube/config. From a3702779a7013d7639aa033678410f62a0e19232 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Mon, 10 Oct 2022 19:15:39 -0700 Subject: [PATCH 005/113] provider: only store resource if spec has changed (#480) * provider: only store resource if spec has changed Leverage the metadata.Generation field to consider whether to update the newly reconciled resource into the watchable map which will trigger translations in the backend. Fixes: https://github.com/envoyproxy/gateway/issues/407 Signed-off-by: Arko Dasgupta --- internal/provider/kubernetes/gateway.go | 31 +++++++++++++++---- internal/provider/kubernetes/httproute.go | 9 +++--- .../provider/kubernetes/kubernetes_test.go | 7 ++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index c6e3a01b9df..7a2a9842604 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -201,11 +201,32 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req status.UpdateGatewayStatusScheduledCondition(&gw, true) // update address field and ready condition status.UpdateGatewayStatusReadyCondition(&gw, svc, deployment) - // publish status - key := utils.NamespacedName(&gw) - r.resources.GatewayStatuses.Store(key, &gw) - r.resources.Gateways.Store(key, &gw) + key := utils.NamespacedName(&gw) + // publish status + // do it inline since this code flow updates the + // Status.Addresses field whereas the message bus / subscriber + // does not. + r.statusUpdater.Send(status.Update{ + NamespacedName: key, + Resource: new(gwapiv1b1.Gateway), + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + g, ok := obj.(*gwapiv1b1.Gateway) + if !ok { + panic(fmt.Sprintf("unsupported object type %T", obj)) + } + gCopy := g.DeepCopy() + gCopy.Status.Conditions = status.MergeConditions(gCopy.Status.Conditions, gw.Status.Conditions...) + gCopy.Status.Addresses = gw.Status.Addresses + return gCopy + + }), + }) + + // only store the resource if it does not exist or it has a newer spec. + if v, ok := r.resources.Gateways.Load(key); !ok || (gw.Generation > v.Generation) { + r.resources.Gateways.Store(key, &gw) + } if key == request.NamespacedName { found = true } @@ -343,8 +364,6 @@ func (r *gatewayReconciler) subscribeAndUpdateStatus(ctx context.Context) { panic(fmt.Sprintf("unsupported object type %T", obj)) } gCopy := g.DeepCopy() - gCopy.Status.Conditions = status.MergeConditions(gCopy.Status.Conditions, val.Status.Conditions...) - gCopy.Status.Addresses = val.Status.Addresses gCopy.Status.Listeners = val.Status.Listeners return gCopy }), diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go index 88eb98a591d..6bae34c938f 100644 --- a/internal/provider/kubernetes/httproute.go +++ b/internal/provider/kubernetes/httproute.go @@ -212,10 +212,11 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, nil } - // Store the httproute in the resource map. - r.resources.HTTPRoutes.Store(routeKey, &route) - log.Info("added httproute to resource map") - + // only store the resource if it does not exist or it has a newer spec. + if v, ok := r.resources.HTTPRoutes.Load(routeKey); !ok || (route.Generation > v.Generation) { + r.resources.HTTPRoutes.Store(routeKey, &route) + log.Info("added httproute to resource map") + } // Get the route's namespace from the cache. nsKey := types.NamespacedName{Name: route.Namespace} ns := new(corev1.Namespace) diff --git a/internal/provider/kubernetes/kubernetes_test.go b/internal/provider/kubernetes/kubernetes_test.go index fec19e3d455..409f7fa1833 100644 --- a/internal/provider/kubernetes/kubernetes_test.go +++ b/internal/provider/kubernetes/kubernetes_test.go @@ -235,7 +235,12 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro return cli.Get(ctx, key, gw) == nil }, defaultWait, defaultTick) gws, _ := resources.Gateways.Load(key) - assert.Equal(t, gw, gws) + // Only check if the spec is equal + // The watchable map will not store a resource + // with an updated status if the spec has not changed + // to eliminate this endless loop: + // reconcile->store->translate->update-status->reconcile + assert.Equal(t, gw.Spec, gws.Spec) } func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { From 8462fe8ab62d2ecdb75cd62ef122d018d7b54c24 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Mon, 10 Oct 2022 19:15:57 -0700 Subject: [PATCH 006/113] remove xds ir listener sort (#537) No longer needed now that order is maintained by using a list, thanks to https://github.com/envoyproxy/gateway/pull/535 Signed-off-by: Arko Dasgupta --- internal/gatewayapi/sort.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/gatewayapi/sort.go b/internal/gatewayapi/sort.go index 035d2f6093b..ecf91bff905 100644 --- a/internal/gatewayapi/sort.go +++ b/internal/gatewayapi/sort.go @@ -44,8 +44,6 @@ func (x XdsIRRoutes) Less(i, j int) bool { // https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteRule func sortXdsIRMap(xdsIR XdsIRMap) { for _, ir := range xdsIR { - ir := ir - sort.SliceStable(ir.HTTP, func(i, j int) bool { return ir.HTTP[i].Name < ir.HTTP[j].Name }) for _, http := range ir.HTTP { // descending order sort.Sort(sort.Reverse(XdsIRRoutes(http.Routes))) From 872c7f5d50be3dcdeedeab67d612b0ecd234ee01 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Tue, 11 Oct 2022 08:55:04 -0700 Subject: [PATCH 007/113] Merges Release Manifests (#510) Signed-off-by: danehans Signed-off-by: danehans --- .github/workflows/release.yaml | 1 - kustomization.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ef0a318ffb0..6760efc924b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -34,7 +34,6 @@ jobs: uses: softprops/action-gh-release@v1 with: files: | - release-artifacts/gatewayapi-crds.yaml release-artifacts/install.yaml diff --git a/kustomization.yaml b/kustomization.yaml index a99c6d192d9..60f835a20fa 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -1,3 +1,4 @@ resources: + - release-artifacts/gatewayapi-crds.yaml - release-artifacts/envoy-gateway.yaml - release-artifacts/infra-manager-rbac.yaml From b9ed6df7cbb730fa363d86ad8476b6f57e88b5b2 Mon Sep 17 00:00:00 2001 From: Shubham Chauhan Date: Tue, 11 Oct 2022 23:32:03 +0530 Subject: [PATCH 008/113] TLS Passthrough support (#402) * TLS Passthrough support This commit adds a tlsroute controller which is further used to configure tls passthrough in envoy. Signed-off-by: Shubham Chauhan * Adding tlsroute experimental crd in testdata update gatewayclass/gateway/httproute experimental CRDs to use standard schemas Signed-off-by: Shubham Chauhan * keep other testdata changes out of this PR Signed-off-by: Shubham Chauhan * added testcases for tlsroutes, include serviceport in irInfraPortName Signed-off-by: Shubham Chauhan * lintfix Signed-off-by: Shubham Chauhan * tlroute kubernetes provider test Signed-off-by: Shubham Chauhan * added xds tls config validate test for passthrough Signed-off-by: Shubham Chauhan * types test tlsroute Signed-off-by: Shubham Chauhan * test fixes Signed-off-by: Shubham Chauhan * xds config tests for tls passthrough Signed-off-by: Shubham Chauhan * increase test coverage Signed-off-by: Shubham Chauhan * testfix Signed-off-by: Shubham Chauhan * separate xds tls listener Signed-off-by: Shubham Chauhan testfix Signed-off-by: Shubham Chauhan * additional xds validate tests Signed-off-by: Shubham Chauhan * tlsroute refgrant test Signed-off-by: Shubham Chauhan * add rbac permissions for tlsroute Signed-off-by: Shubham Chauhan * updates post rebase Signed-off-by: Shubham Chauhan * add status updater, gateway watcher for tlsroute Signed-off-by: Shubham Chauhan * add status update framework for tlsroute Signed-off-by: Shubham Chauhan * lintfix, testfix, fix post rebase Signed-off-by: Shubham Chauhan * yet another lintfix Signed-off-by: Shubham Chauhan * refactor tlslistener/route -> tcplistener/route, xds updates Signed-off-by: Shubham Chauhan * missed a file Signed-off-by: Shubham Chauhan * lintfix Signed-off-by: Shubham Chauhan * rebase, review comments Signed-off-by: Shubham Chauhan * minor testfix Signed-off-by: Shubham Chauhan * more Signed-off-by: Shubham Chauhan * review comments, status deepcopy, check routes in ns Signed-off-by: Shubham Chauhan * revert bad import, testfix, new test Signed-off-by: Shubham Chauhan * rev sort Signed-off-by: Shubham Chauhan Signed-off-by: Shubham Chauhan --- internal/cmd/server.go | 2 + internal/envoygateway/scheme.go | 6 +- internal/gatewayapi/contexts.go | 215 ++++++- internal/gatewayapi/helpers.go | 11 +- internal/gatewayapi/helpers_v1alpha2.go | 136 +++++ internal/gatewayapi/runner/runner.go | 8 + ...ith-invalid-allowed-tls-route-kind.in.yaml | 35 ++ ...th-invalid-allowed-tls-route-kind.out.yaml | 72 +++ ...lid-tls-configuration-invalid-mode.in.yaml | 1 + ...id-tls-configuration-invalid-mode.out.yaml | 1 + ...with-tls-terminate-and-passthrough.in.yaml | 70 +++ ...ith-tls-terminate-and-passthrough.out.yaml | 154 +++++ ...ener-with-valid-tls-configuration.out.yaml | 2 +- ...nd-tlsroute-same-hostname-and-port.in.yaml | 56 ++ ...d-tlsroute-same-hostname-and-port.out.yaml | 120 ++++ ...with-invalid-backend-ref-bad-port.out.yaml | 2 +- ...-invalid-backend-ref-invalid-kind.out.yaml | 2 +- ...-with-invalid-backend-ref-no-port.out.yaml | 2 +- ...th-invalid-backend-ref-no-service.out.yaml | 2 +- .../tlsroute-attaching-to-gateway.in.yaml | 32 ++ .../tlsroute-attaching-to-gateway.out.yaml | 84 +++ ...ing-to-gateway-with-incorrect-mode.in.yaml | 31 + ...ng-to-gateway-with-incorrect-mode.out.yaml | 67 +++ ...-attaching-to-gateway-with-no-mode.in.yaml | 29 + ...attaching-to-gateway-with-no-mode.out.yaml | 65 +++ ...ther-namespace-allowed-by-refgrant.in.yaml | 57 ++ ...her-namespace-allowed-by-refgrant.out.yaml | 85 +++ ...ner-both-passthrough-and-cert-data.in.yaml | 44 ++ ...er-both-passthrough-and-cert-data.out.yaml | 70 +++ ...ute-with-partial-wildcard-hostname.in.yaml | 52 ++ ...te-with-partial-wildcard-hostname.out.yaml | 68 +++ internal/gatewayapi/translator.go | 487 +++++++++++++--- internal/ir/infra.go | 3 + internal/ir/xds.go | 130 ++++- internal/ir/xds_test.go | 86 ++- internal/ir/zz_generated.deepcopy.go | 62 ++ internal/message/types.go | 14 + internal/message/types_test.go | 32 ++ .../provider/kubernetes/config/rbac/role.yaml | 2 + internal/provider/kubernetes/helpers.go | 76 +++ internal/provider/kubernetes/httproute.go | 57 +- .../provider/kubernetes/httproute_test.go | 2 +- internal/provider/kubernetes/kubernetes.go | 4 + .../provider/kubernetes/kubernetes_test.go | 206 +++++-- internal/provider/kubernetes/rbac.go | 4 +- .../in/tlsroute-experimental-crd.yaml | 542 ++++++++++++++++++ internal/provider/kubernetes/tlsroute.go | 326 +++++++++++ internal/status/status.go | 8 + internal/xds/translator/cluster.go | 6 +- internal/xds/translator/listener.go | 71 ++- .../in/xds-ir/tls-route-passthrough.yaml | 12 + .../tls-route-passthrough.clusters.yaml | 23 + .../tls-route-passthrough.listeners.yaml | 19 + .../xds-ir/tls-route-passthrough.routes.yaml | 1 + internal/xds/translator/translator.go | 18 +- internal/xds/translator/translator_test.go | 3 + 56 files changed, 3517 insertions(+), 258 deletions(-) create mode 100644 internal/gatewayapi/helpers_v1alpha2.go create mode 100644 internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.in.yaml create mode 100644 internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml create mode 100644 internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.in.yaml create mode 100644 internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml create mode 100644 internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.in.yaml create mode 100644 internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml create mode 100644 internal/provider/kubernetes/helpers.go create mode 100644 internal/provider/kubernetes/testdata/in/tlsroute-experimental-crd.yaml create mode 100644 internal/provider/kubernetes/tlsroute.go create mode 100644 internal/xds/translator/testdata/in/xds-ir/tls-route-passthrough.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.routes.yaml diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 3a7bf625116..052d0229e49 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -153,6 +153,8 @@ func setupRunners(cfg *config.Server) error { pResources.Namespaces.Close() pResources.GatewayStatuses.Close() pResources.HTTPRouteStatuses.Close() + pResources.TLSRoutes.Close() + pResources.TLSRouteStatuses.Close() xdsIR.Close() infraIR.Close() xds.Close() diff --git a/internal/envoygateway/scheme.go b/internal/envoygateway/scheme.go index e4971c7c7ed..31fc950c4cd 100644 --- a/internal/envoygateway/scheme.go +++ b/internal/envoygateway/scheme.go @@ -3,7 +3,8 @@ package envoygateway import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/api/config/v1alpha1" ) @@ -28,6 +29,9 @@ func init() { if err := gwapiv1b1.AddToScheme(scheme); err != nil { panic(err) } + if err := gwapiv1a2.AddToScheme(scheme); err != nil { + panic(err) + } } // GetScheme returns a scheme with types supported by the Kubernetes provider. diff --git a/internal/gatewayapi/contexts.go b/internal/gatewayapi/contexts.go index 3b1aec98c56..cb5d56a2274 100644 --- a/internal/gatewayapi/contexts.go +++ b/internal/gatewayapi/contexts.go @@ -7,6 +7,8 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" egv1alpha1 "github.com/envoyproxy/gateway/api/config/v1alpha1" @@ -20,6 +22,10 @@ type GatewayContext struct { listeners []*ListenerContext } +// GetListenerContext returns the ListenerContext with listenerName. +// If the listener exists in the Gateway Spec but NOT yet in the GatewayContext, +// this creates a new ListenerContext for the listener and attaches it to the +// GatewayContext. func (g *GatewayContext) GetListenerContext(listenerName v1beta1.SectionName) *ListenerContext { if g.listeners == nil { g.listeners = make([]*ListenerContext, 0) @@ -169,6 +175,30 @@ func (l *ListenerContext) SetTLSSecret(tlsSecret *v1.Secret) { l.tlsSecret = tlsSecret } +// RouteContext represents a generic Route object (HTTPRoute, TLSRoute, etc.) +// that can reference Gateway objects. +type RouteContext interface { + client.Object + + // GetRouteType returns the Kind of the Route object, HTTPRoute, + // TLSRoute, TCPRoute, UDPRoute etc. + GetRouteType() string + + // TODO: [v1alpha2-v1beta1] This should not be required once all Route + // objects being implemented are of type v1beta1. + // GetHostnames returns the hosts targeted by the Route object. + GetHostnames() []string + + // TODO: [v1alpha2-v1beta1] This should not be required once all Route + // objects being implemented are of type v1beta1. + // GetParentReferences returns the ParentReference of the Route object. + GetParentReferences() []v1beta1.ParentReference + + // GetRouteParentContext returns RouteParentContext by using the Route + // objects' ParentReference. + GetRouteParentContext(forParentRef v1beta1.ParentReference) *RouteParentContext +} + // HTTPRouteContext wraps an HTTPRoute and provides helper methods for // accessing the route's parents. type HTTPRouteContext struct { @@ -177,6 +207,22 @@ type HTTPRouteContext struct { parentRefs map[v1beta1.ParentReference]*RouteParentContext } +func (h *HTTPRouteContext) GetRouteType() string { + return KindHTTPRoute +} + +func (h *HTTPRouteContext) GetHostnames() []string { + hostnames := make([]string, len(h.Spec.Hostnames)) + for idx, s := range h.Spec.Hostnames { + hostnames[idx] = string(s) + } + return hostnames +} + +func (h *HTTPRouteContext) GetParentReferences() []v1beta1.ParentReference { + return h.Spec.ParentRefs +} + func (h *HTTPRouteContext) GetRouteParentContext(forParentRef v1beta1.ParentReference) *RouteParentContext { if h.parentRefs == nil { h.parentRefs = make(map[v1beta1.ParentReference]*RouteParentContext) @@ -217,20 +263,109 @@ func (h *HTTPRouteContext) GetRouteParentContext(forParentRef v1beta1.ParentRefe ctx := &RouteParentContext{ ParentReference: parentRef, - route: h.HTTPRoute, + httpRoute: h.HTTPRoute, routeParentStatusIdx: routeParentStatusIdx, } h.parentRefs[forParentRef] = ctx return ctx } +// TLSRouteContext wraps a TLSRoute and provides helper methods for +// accessing the route's parents. +type TLSRouteContext struct { + *v1alpha2.TLSRoute + + parentRefs map[v1beta1.ParentReference]*RouteParentContext +} + +func (t *TLSRouteContext) GetRouteType() string { + return KindTLSRoute +} + +func (t *TLSRouteContext) GetHostnames() []string { + hostnames := make([]string, len(t.Spec.Hostnames)) + for idx, s := range t.Spec.Hostnames { + hostnames[idx] = string(s) + } + return hostnames +} + +func (t *TLSRouteContext) GetParentReferences() []v1beta1.ParentReference { + parentReferences := make([]v1beta1.ParentReference, len(t.Spec.ParentRefs)) + for idx, p := range t.Spec.ParentRefs { + parentReferences[idx] = UpgradeParentReference(p) + } + return parentReferences +} + +func (t *TLSRouteContext) GetRouteParentContext(forParentRef v1beta1.ParentReference) *RouteParentContext { + if t.parentRefs == nil { + t.parentRefs = make(map[v1beta1.ParentReference]*RouteParentContext) + } + + if ctx := t.parentRefs[forParentRef]; ctx != nil { + return ctx + } + + var parentRef *v1beta1.ParentReference + for i, p := range t.Spec.ParentRefs { + p := UpgradeParentReference(p) + if reflect.DeepEqual(p, forParentRef) { + upgraded := UpgradeParentReference(t.Spec.ParentRefs[i]) + parentRef = &upgraded + break + } + } + if parentRef == nil { + panic("parentRef not found") + } + + routeParentStatusIdx := -1 + for i := range t.Status.Parents { + p := UpgradeParentReference(t.Status.Parents[i].ParentRef) + defaultNamespace := v1beta1.Namespace(metav1.NamespaceDefault) + if forParentRef.Namespace == nil { + forParentRef.Namespace = &defaultNamespace + } + if p.Namespace == nil { + p.Namespace = &defaultNamespace + } + if reflect.DeepEqual(p, forParentRef) { + routeParentStatusIdx = i + break + } + } + if routeParentStatusIdx == -1 { + rParentStatus := v1alpha2.RouteParentStatus{ + // TODO: get this value from the config + ControllerName: v1alpha2.GatewayController(egv1alpha1.GatewayControllerName), + ParentRef: DowngradeParentReference(forParentRef), + } + t.Status.Parents = append(t.Status.Parents, rParentStatus) + routeParentStatusIdx = len(t.Status.Parents) - 1 + } + + ctx := &RouteParentContext{ + ParentReference: parentRef, + + tlsRoute: t.TLSRoute, + routeParentStatusIdx: routeParentStatusIdx, + } + t.parentRefs[forParentRef] = ctx + return ctx +} + // RouteParentContext wraps a ParentReference and provides helper methods for // setting conditions and other status information on the associated -// HTTPRoute, etc. +// HTTPRoute, TLSRoute etc. type RouteParentContext struct { *v1beta1.ParentReference - route *v1beta1.HTTPRoute + // TODO: [v1alpha2-v1beta1] This can probably be replaced with + // a single field pointing to *v1beta1.RouteStatus. + httpRoute *v1beta1.HTTPRoute + tlsRoute *v1alpha2.TLSRoute + routeParentStatusIdx int listeners []*ListenerContext } @@ -239,43 +374,77 @@ func (r *RouteParentContext) SetListeners(listeners ...*ListenerContext) { r.listeners = append(r.listeners, listeners...) } -func (r *RouteParentContext) SetCondition(conditionType v1beta1.RouteConditionType, status metav1.ConditionStatus, reason v1beta1.RouteConditionReason, message string) { +func (r *RouteParentContext) SetCondition(route RouteContext, conditionType v1beta1.RouteConditionType, status metav1.ConditionStatus, reason v1beta1.RouteConditionReason, message string) { cond := metav1.Condition{ Type: string(conditionType), Status: status, Reason: string(reason), Message: message, - ObservedGeneration: r.route.Generation, + ObservedGeneration: route.GetGeneration(), LastTransitionTime: metav1.NewTime(time.Now()), } idx := -1 - for i, existing := range r.route.Status.Parents[r.routeParentStatusIdx].Conditions { - if existing.Type == cond.Type { - // return early if the condition is unchanged - if existing.Status == cond.Status && - existing.Reason == cond.Reason && - existing.Message == cond.Message { - return + switch route.GetRouteType() { + case KindHTTPRoute: + for i, existing := range r.httpRoute.Status.Parents[r.routeParentStatusIdx].Conditions { + if existing.Type == cond.Type { + // return early if the condition is unchanged + if existing.Status == cond.Status && + existing.Reason == cond.Reason && + existing.Message == cond.Message { + return + } + idx = i + break } - idx = i - break } - } - if idx > -1 { - r.route.Status.Parents[r.routeParentStatusIdx].Conditions[idx] = cond - } else { - r.route.Status.Parents[r.routeParentStatusIdx].Conditions = append(r.route.Status.Parents[r.routeParentStatusIdx].Conditions, cond) + if idx > -1 { + r.httpRoute.Status.Parents[r.routeParentStatusIdx].Conditions[idx] = cond + } else { + r.httpRoute.Status.Parents[r.routeParentStatusIdx].Conditions = append(r.httpRoute.Status.Parents[r.routeParentStatusIdx].Conditions, cond) + } + case KindTLSRoute: + for i, existing := range r.tlsRoute.Status.Parents[r.routeParentStatusIdx].Conditions { + if existing.Type == cond.Type { + // return early if the condition is unchanged + if existing.Status == cond.Status && + existing.Reason == cond.Reason && + existing.Message == cond.Message { + return + } + idx = i + break + } + } + + if idx > -1 { + r.tlsRoute.Status.Parents[r.routeParentStatusIdx].Conditions[idx] = cond + } else { + r.tlsRoute.Status.Parents[r.routeParentStatusIdx].Conditions = append(r.tlsRoute.Status.Parents[r.routeParentStatusIdx].Conditions, cond) + } } } -func (r *RouteParentContext) ResetConditions() { - r.route.Status.Parents[r.routeParentStatusIdx].Conditions = make([]metav1.Condition, 0) +func (r *RouteParentContext) ResetConditions(route RouteContext) { + switch route.GetRouteType() { + case KindHTTPRoute: + r.httpRoute.Status.Parents[r.routeParentStatusIdx].Conditions = make([]metav1.Condition, 0) + case KindTLSRoute: + r.tlsRoute.Status.Parents[r.routeParentStatusIdx].Conditions = make([]metav1.Condition, 0) + } } -func (r *RouteParentContext) IsAccepted() bool { - for _, cond := range r.route.Status.Parents[r.routeParentStatusIdx].Conditions { +func (r *RouteParentContext) IsAccepted(route RouteContext) bool { + var conditions []metav1.Condition + switch route.GetRouteType() { + case KindHTTPRoute: + conditions = r.httpRoute.Status.Parents[r.routeParentStatusIdx].Conditions + case KindTLSRoute: + conditions = r.tlsRoute.Status.Parents[r.routeParentStatusIdx].Conditions + } + for _, cond := range conditions { if cond.Type == string(v1beta1.RouteConditionAccepted) && cond.Status == metav1.ConditionTrue { return true } diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index 4b54b610716..a9c2a93a4d4 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -27,6 +27,11 @@ func FromNamespacesPtr(fromNamespaces v1beta1.FromNamespaces) *v1beta1.FromNames return &fromNamespaces } +func SectionNamePtr(name string) *v1beta1.SectionName { + sectionName := v1beta1.SectionName(name) + return §ionName +} + func StringPtr(val string) *string { return &val } @@ -138,9 +143,9 @@ func HasReadyListener(listeners []*ListenerContext) bool { return false } -// ComputeHosts returns a list of the intersecting hostnames between the route +// computeHosts returns a list of the intersecting hostnames between the route // and the listener. -func ComputeHosts(routeHostnames []v1beta1.Hostname, listenerHostname *v1beta1.Hostname) []string { +func computeHosts(routeHostnames []string, listenerHostname *v1beta1.Hostname) []string { var listenerHostnameVal string if listenerHostname != nil { listenerHostnameVal = string(*listenerHostname) @@ -159,7 +164,7 @@ func ComputeHosts(routeHostnames []v1beta1.Hostname, listenerHostname *v1beta1.H var hostnames []string for i := range routeHostnames { - routeHostname := string(routeHostnames[i]) + routeHostname := routeHostnames[i] // TODO ensure routeHostname is a valid hostname diff --git a/internal/gatewayapi/helpers_v1alpha2.go b/internal/gatewayapi/helpers_v1alpha2.go new file mode 100644 index 00000000000..c4b2153f2a2 --- /dev/null +++ b/internal/gatewayapi/helpers_v1alpha2.go @@ -0,0 +1,136 @@ +// Portions of this code are based on code from Contour. + +package gatewayapi + +import ( + "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// TODO: [v1alpha2-v1beta1] +// This file can be removed once TLSRoute graduates to v1beta1. + +func GroupPtrV1Alpha2(group string) *v1alpha2.Group { + gwGroup := v1alpha2.Group(group) + return &gwGroup +} + +func KindPtrV1Alpha2(kind string) *v1alpha2.Kind { + gwKind := v1alpha2.Kind(kind) + return &gwKind +} + +func NamespacePtrV1Alpha2(namespace string) *v1alpha2.Namespace { + gwNamespace := v1alpha2.Namespace(namespace) + return &gwNamespace +} + +func SectionNamePtrV1Alpha2(sectionName string) *v1alpha2.SectionName { + gwSectionName := v1alpha2.SectionName(sectionName) + return &gwSectionName +} + +func PortNumPtrV1Alpha2(port int) *v1alpha2.PortNumber { + pn := v1alpha2.PortNumber(port) + return &pn +} + +func UpgradeParentReferences(old []v1alpha2.ParentReference) []v1beta1.ParentReference { + newParentReferences := make([]v1beta1.ParentReference, len(old)) + for i, o := range old { + newParentReferences[i] = UpgradeParentReference(o) + } + return newParentReferences +} + +// UpgradeParentReference converts v1alpha2.ParentReference to v1beta1.ParentReference +func UpgradeParentReference(old v1alpha2.ParentReference) v1beta1.ParentReference { + upgraded := v1beta1.ParentReference{} + + if old.Group != nil { + upgraded.Group = GroupPtr(string(*old.Group)) + } + + if old.Kind != nil { + upgraded.Kind = KindPtr(string(*old.Kind)) + } + + if old.Namespace != nil { + upgraded.Namespace = NamespacePtr(string(*old.Namespace)) + } + + upgraded.Name = v1beta1.ObjectName(old.Name) + + if old.SectionName != nil { + upgraded.SectionName = SectionNamePtr(string(*old.SectionName)) + } + + if old.Port != nil { + upgraded.Port = PortNumPtr(int32(*old.Port)) + } + + return upgraded +} + +func DowngradeParentReference(old v1beta1.ParentReference) v1alpha2.ParentReference { + downgraded := v1alpha2.ParentReference{} + + if old.Group != nil { + downgraded.Group = GroupPtrV1Alpha2(string(*old.Group)) + } + + if old.Kind != nil { + downgraded.Kind = KindPtrV1Alpha2(string(*old.Kind)) + } + + if old.Namespace != nil { + downgraded.Namespace = NamespacePtrV1Alpha2(string(*old.Namespace)) + } + + downgraded.Name = v1alpha2.ObjectName(old.Name) + + if old.SectionName != nil { + downgraded.SectionName = SectionNamePtrV1Alpha2(string(*old.SectionName)) + } + + if old.Port != nil { + downgraded.Port = PortNumPtrV1Alpha2(int(*old.Port)) + } + + return downgraded +} + +func UpgradeRouteParentStatuses(routeParentStatuses []v1alpha2.RouteParentStatus) []v1beta1.RouteParentStatus { + var res []v1beta1.RouteParentStatus + + for _, rps := range routeParentStatuses { + res = append(res, v1beta1.RouteParentStatus{ + ParentRef: UpgradeParentReference(rps.ParentRef), + ControllerName: v1beta1.GatewayController(rps.ControllerName), + Conditions: rps.Conditions, + }) + } + + return res +} + +func DowngradeRouteParentStatuses(routeParentStatuses []v1beta1.RouteParentStatus) []v1alpha2.RouteParentStatus { + var res []v1alpha2.RouteParentStatus + + for _, rps := range routeParentStatuses { + res = append(res, v1alpha2.RouteParentStatus{ + ParentRef: DowngradeParentReference(rps.ParentRef), + ControllerName: v1alpha2.GatewayController(rps.ControllerName), + Conditions: rps.Conditions, + }) + } + + return res +} + +func NamespaceDerefOrAlpha(namespace *v1alpha2.Namespace, defaultNamespace string) string { + if namespace != nil && *namespace != "" { + return string(*namespace) + } + return defaultNamespace +} diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index 05c5929a7aa..d4793d5ecdb 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -44,6 +44,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { gatewayClassesCh := r.ProviderResources.GatewayClasses.Subscribe(ctx) gatewaysCh := r.ProviderResources.Gateways.Subscribe(ctx) httpRoutesCh := r.ProviderResources.HTTPRoutes.Subscribe(ctx) + tlsRoutesCh := r.ProviderResources.TLSRoutes.Subscribe(ctx) servicesCh := r.ProviderResources.Services.Subscribe(ctx) namespacesCh := r.ProviderResources.Namespaces.Subscribe(ctx) @@ -54,6 +55,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { case <-gatewayClassesCh: case <-gatewaysCh: case <-httpRoutesCh: + case <-tlsRoutesCh: case <-servicesCh: case <-namespacesCh: } @@ -61,6 +63,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Load all resources required for translation in.Gateways = r.ProviderResources.GetGateways() in.HTTPRoutes = r.ProviderResources.GetHTTPRoutes() + in.TLSRoutes = r.ProviderResources.GetTLSRoutes() in.Services = r.ProviderResources.GetServices() in.Namespaces = r.ProviderResources.GetNamespaces() gatewayClasses := r.ProviderResources.GetGatewayClasses() @@ -99,6 +102,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { newKeys = append(newKeys, key) } } + for key, val := range result.XdsIR { if err := val.Validate(); err != nil { r.Logger.Error(err, "unable to validate xds ir, skipped sending it") @@ -124,6 +128,10 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { key := utils.NamespacedName(httpRoute) r.ProviderResources.HTTPRouteStatuses.Store(key, httpRoute) } + for _, tlsRoute := range result.TLSRoutes { + key := utils.NamespacedName(tlsRoute) + r.ProviderResources.TLSRouteStatuses.Store(key, tlsRoute) + } } } r.Logger.Info("shutting down") diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.in.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.in.yaml new file mode 100644 index 00000000000..c5762afbaff --- /dev/null +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.in.yaml @@ -0,0 +1,35 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + hostname: foo.com + protocol: TLS + port: 80 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + kinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml new file mode 100644 index 00000000000..59c61cc7162 --- /dev/null +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml @@ -0,0 +1,72 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + port: 80 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + kinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + status: + listeners: + - name: tls + attachedRoutes: 0 + conditions: + - type: ResolvedRefs + status: "False" + reason: InvalidRouteKinds + message: "Kind is not supported, kind must be TLSRoute" + - type: Ready + status: "False" + reason: Invalid + message: Listener is invalid, see other Conditions for details. +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref +xdsIR: + envoy-gateway-gateway-1: {} +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.in.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.in.yaml index c17a2ef492e..a4793ddeff2 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.in.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.in.yaml @@ -9,6 +9,7 @@ gateways: listeners: - name: tls protocol: HTTPS + hostname: foo.com port: 443 allowedRoutes: namespaces: diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml index 40a696060d1..13c19dc555f 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml @@ -9,6 +9,7 @@ gateways: listeners: - name: tls protocol: HTTPS + hostname: foo.com port: 443 allowedRoutes: namespaces: diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.in.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.in.yaml new file mode 100644 index 00000000000..bf254c658b1 --- /dev/null +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.in.yaml @@ -0,0 +1,70 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls-passthrough + protocol: TLS + port: 90 + hostname: foo.com + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + - name: tls-terminate + protocol: HTTPS + port: 443 + hostname: foo.com + tls: + mode: Terminate + certificateRefs: + - name: tls-secret-1 + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-2 + port: 8080 +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 +secrets: + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: tls-secret-1 + type: kubernetes.io/tls + data: + tls.crt: Zm9vCg== + tls.key: YmFyCg== diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml new file mode 100644 index 00000000000..f47a7e3a1f2 --- /dev/null +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml @@ -0,0 +1,154 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls-passthrough + protocol: TLS + port: 90 + hostname: foo.com + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + - name: tls-terminate + protocol: HTTPS + port: 443 + hostname: foo.com + tls: + mode: Terminate + certificateRefs: + - name: tls-secret-1 + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls-passthrough + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready + - name: tls-terminate + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-2 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + http: + - name: envoy-gateway-gateway-1-tls-terminate + address: 0.0.0.0 + port: 10443 + hostnames: + - "foo.com" + tls: + serverCertificate: Zm9vCg== + privateKey: YmFyCg== + routes: + - name: default-httproute-1-rule-0-match-0-foo.com + pathMatch: + prefix: "/" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + tcp: + - name: envoy-gateway-gateway-1-tls-passthrough + address: 0.0.0.0 + port: 10090 + tls: + snis: + - "foo.com" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" + ports: + - name: tls-passthrough + protocol: "TLS" + servicePort: 90 + containerPort: 10090 + - name: tls-terminate + protocol: "HTTPS" + servicePort: 443 + containerPort: 10443 diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml index c13cd35b074..ecd190d5e18 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml @@ -81,8 +81,8 @@ infraIR: proxy: metadata: labels: - gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 image: envoyproxy/envoy:v1.23-latest listeners: diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.in.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.in.yaml new file mode 100644 index 00000000000..e3bfa502356 --- /dev/null +++ b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.in.yaml @@ -0,0 +1,56 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http-1 + protocol: HTTP + port: 80 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + - name: tls-1 + protocol: TLS + tls: + mode: Passthrough + port: 80 + hostname: foo.com + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml new file mode 100644 index 00000000000..f0c6ae9bc5f --- /dev/null +++ b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml @@ -0,0 +1,120 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http-1 + protocol: HTTP + port: 80 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + - name: tls-1 + protocol: TLS + tls: + mode: Passthrough + port: 80 + hostname: foo.com + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: http-1 + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + conditions: + - type: Conflicted + status: "True" + reason: HostnameConflict + message: All listeners for a given port must use a unique hostname + - type: Ready + status: "False" + reason: Invalid + message: Listener is invalid, see other Conditions for details. + - name: tls-1 + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + conditions: + - type: Conflicted + status: "True" + reason: HostnameConflict + message: All listeners for a given port must use a unique hostname + - type: Ready + status: "False" + reason: Invalid + message: Listener is invalid, see other Conditions for details. +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref + +xdsIR: + envoy-gateway-gateway-1: {} +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml index 18fc9ff2c89..3218c2096ed 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml @@ -79,8 +79,8 @@ infraIR: proxy: metadata: labels: - gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 image: envoyproxy/envoy:v1.23-latest listeners: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml index a0b76319001..7b0ad031a99 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml @@ -80,8 +80,8 @@ infraIR: proxy: metadata: labels: - gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 image: envoyproxy/envoy:v1.23-latest listeners: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml index 634ed1b1bcb..5e41249f9d5 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml @@ -78,8 +78,8 @@ infraIR: proxy: metadata: labels: - gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 image: envoyproxy/envoy:v1.23-latest listeners: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml index 4dfdd8dd667..27f0f389c47 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml @@ -79,8 +79,8 @@ infraIR: proxy: metadata: labels: - gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 image: envoyproxy/envoy:v1.23-latest listeners: diff --git a/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.in.yaml b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.in.yaml new file mode 100644 index 00000000000..71db6c0ece6 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.in.yaml @@ -0,0 +1,32 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + port: 90 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml new file mode 100644 index 00000000000..9a82becdbce --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml @@ -0,0 +1,84 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + port: 90 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + tcp: + - name: envoy-gateway-gateway-1-tls + address: 0.0.0.0 + port: 10090 + tls: + snis: + - foo.com + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" + ports: + - name: tls + protocol: "TLS" + servicePort: 90 + containerPort: 10090 diff --git a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.in.yaml b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.in.yaml new file mode 100644 index 00000000000..d8a62c2adf3 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.in.yaml @@ -0,0 +1,31 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + tls: + mode: Terminate + port: 90 + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml new file mode 100644 index 00000000000..90c62500616 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml @@ -0,0 +1,67 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + tls: + mode: Terminate + port: 90 + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 0 + conditions: + - type: Ready + status: "False" + reason: UnsupportedTLSMode + message: TLS Terminate mode is not supported, TLS mode must be Passthrough. +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref +xdsIR: + envoy-gateway-gateway-1: {} +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" diff --git a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.in.yaml b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.in.yaml new file mode 100644 index 00000000000..38395404cd8 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.in.yaml @@ -0,0 +1,29 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 90 + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml new file mode 100644 index 00000000000..dbc22d0a5d5 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml @@ -0,0 +1,65 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 90 + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 0 + conditions: + - type: Ready + status: "False" + reason: Invalid + message: Listener must have TLS set when protocol is TLS. +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref +xdsIR: + envoy-gateway-gateway-1: {} +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" diff --git a/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.in.yaml b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.in.yaml new file mode 100644 index 00000000000..d18aca10b82 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.in.yaml @@ -0,0 +1,57 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + port: 90 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + namespace: test-service-namespace + port: 8080 +services: + - apiVersion: v1 + kind: Service + metadata: + namespace: test-service-namespace + name: service-1 + spec: + clusterIP: 7.7.7.7 + ports: + - port: 8080 +referenceGrants: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: ReferenceGrant + metadata: + namespace: test-service-namespace + name: referencegrant-1 + spec: + from: + - group: gateway.networking.k8s.io + kind: TLSRoute + namespace: default + to: + - group: "" + kind: Service diff --git a/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml new file mode 100644 index 00000000000..9bc5dc2c00d --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml @@ -0,0 +1,85 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + port: 90 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + namespace: test-service-namespace + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + tcp: + - name: envoy-gateway-gateway-1-tls + address: 0.0.0.0 + port: 10090 + tls: + snis: + - foo.com + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" + ports: + - name: tls + protocol: "TLS" + servicePort: 90 + containerPort: 10090 diff --git a/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.in.yaml b/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.in.yaml new file mode 100644 index 00000000000..81afa2331b1 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.in.yaml @@ -0,0 +1,44 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + tls: + mode: Passthrough + certificateRefs: + - name: tls-secret-1 + port: 90 + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 +secrets: + - apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: tls-secret-1 + type: kubernetes.io/tls + data: + tls.crt: Zm9vCg== + tls.key: YmFyCg== diff --git a/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml new file mode 100644 index 00000000000..b8b13220444 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml @@ -0,0 +1,70 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + hostname: foo.com + tls: + mode: Passthrough + certificateRefs: + - name: tls-secret-1 + port: 90 + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 0 + conditions: + - type: Ready + status: "False" + reason: Invalid + message: Listener must not have TLS certificate refs set for TLS mode Passthrough +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref +xdsIR: + envoy-gateway-gateway-1: {} +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" diff --git a/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml b/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml new file mode 100644 index 00000000000..ac35ef26721 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml @@ -0,0 +1,52 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + # TODO: add test for partial wildcard + # - name: tls-1 + # protocol: TLS + # hostname: "*w.example.com" + # port: 90 + # tls: + # mode: Passthrough + # allowedRoutes: + # namespaces: + # from: All + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + namespace: test-service-namespace + port: 8080 +services: + - apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-1 + spec: + clusterIP: 7.7.7.7 + ports: + - port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml new file mode 100644 index 00000000000..feb5f807144 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml @@ -0,0 +1,68 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 0 + conditions: + - type: Ready + status: "False" + reason: Invalid + message: Hostname must not be empty with TLS mode Passthrough. +tlsRoutes: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + namespace: test-service-namespace + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: NoReadyListeners + message: There are no ready listeners for this parent ref +xdsIR: + envoy-gateway-gateway-1: {} +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index edf0bd3dc5d..71e3ef9a115 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -19,6 +19,7 @@ import ( const ( KindGateway = "Gateway" KindHTTPRoute = "HTTPRoute" + KindTLSRoute = "TLSRoute" KindService = "Service" KindSecret = "Secret" @@ -45,6 +46,7 @@ type InfraIRMap map[string]*ir.Infra type Resources struct { Gateways []*v1beta1.Gateway HTTPRoutes []*v1beta1.HTTPRoute + TLSRoutes []*v1alpha2.TLSRoute ReferenceGrants []*v1alpha2.ReferenceGrant Namespaces []*v1.Namespace Services []*v1.Service @@ -90,11 +92,14 @@ type Translator struct { type TranslateResult struct { Gateways []*v1beta1.Gateway HTTPRoutes []*v1beta1.HTTPRoute + TLSRoutes []*v1alpha2.TLSRoute XdsIR XdsIRMap InfraIR InfraIRMap } -func newTranslateResult(gateways []*GatewayContext, httpRoutes []*HTTPRouteContext, xdsIR XdsIRMap, infraIR InfraIRMap) *TranslateResult { +func newTranslateResult(gateways []*GatewayContext, + httpRoutes []*HTTPRouteContext, tlsRoutes []*TLSRouteContext, + xdsIR XdsIRMap, infraIR InfraIRMap) *TranslateResult { translateResult := &TranslateResult{ XdsIR: xdsIR, InfraIR: infraIR, @@ -106,6 +111,9 @@ func newTranslateResult(gateways []*GatewayContext, httpRoutes []*HTTPRouteConte for _, httpRoute := range httpRoutes { translateResult.HTTPRoutes = append(translateResult.HTTPRoutes, httpRoute.HTTPRoute) } + for _, tlsRoute := range tlsRoutes { + translateResult.TLSRoutes = append(translateResult.TLSRoutes, tlsRoute.TLSRoute) + } return translateResult } @@ -123,10 +131,13 @@ func (t *Translator) Translate(resources *Resources) *TranslateResult { // Process all relevant HTTPRoutes. httpRoutes := t.ProcessHTTPRoutes(resources.HTTPRoutes, gateways, resources, xdsIR) + // Process all relevant TLSRoutes. + tlsRoutes := t.ProcessTLSRoutes(resources.TLSRoutes, gateways, resources, xdsIR) + // Sort xdsIR based on the Gateway API spec sortXdsIRMap(xdsIR) - return newTranslateResult(gateways, httpRoutes, xdsIR, infraIR) + return newTranslateResult(gateways, httpRoutes, tlsRoutes, xdsIR, infraIR) } func (t *Translator) GetRelevantGateways(gateways []*v1beta1.Gateway) []*GatewayContext { @@ -246,6 +257,33 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap for _, listener := range gateway.listeners { // Process protocol & supported kinds switch listener.Protocol { + case v1beta1.TLSProtocolType: + if listener.AllowedRoutes == nil || len(listener.AllowedRoutes.Kinds) == 0 { + listener.SetSupportedKinds(v1beta1.RouteGroupKind{Group: GroupPtr(v1beta1.GroupName), Kind: KindTLSRoute}) + } else { + for _, kind := range listener.AllowedRoutes.Kinds { + if kind.Group != nil && string(*kind.Group) != v1beta1.GroupName { + listener.SetCondition( + v1beta1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + v1beta1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("Group is not supported, group must be %s", v1beta1.GroupName), + ) + continue + } + + if kind.Kind != KindTLSRoute { + listener.SetCondition( + v1beta1.ListenerConditionResolvedRefs, + metav1.ConditionFalse, + v1beta1.ListenerReasonInvalidRouteKinds, + fmt.Sprintf("Kind is not supported, kind must be %s", KindTLSRoute), + ) + continue + } + listener.SetSupportedKinds(kind) + } + } case v1beta1.HTTPProtocolType, v1beta1.HTTPSProtocolType: if listener.AllowedRoutes == nil || len(listener.AllowedRoutes.Kinds) == 0 { listener.SetSupportedKinds(v1beta1.RouteGroupKind{Group: GroupPtr(v1beta1.GroupName), Kind: KindHTTPRoute}) @@ -258,6 +296,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap v1beta1.ListenerReasonInvalidRouteKinds, fmt.Sprintf("Group is not supported, group must be %s", v1beta1.GroupName), ) + continue } if kind.Kind != KindHTTPRoute { @@ -267,7 +306,9 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap v1beta1.ListenerReasonInvalidRouteKinds, fmt.Sprintf("Kind is not supported, kind must be %s", KindHTTPRoute), ) + continue } + listener.SetSupportedKinds(kind) } } default: @@ -432,6 +473,49 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap } listener.SetTLSSecret(secret) + case v1beta1.TLSProtocolType: + if listener.TLS == nil { + listener.SetCondition( + v1beta1.ListenerConditionReady, + metav1.ConditionFalse, + v1beta1.ListenerReasonInvalid, + fmt.Sprintf("Listener must have TLS set when protocol is %s.", listener.Protocol), + ) + break + } + + if listener.TLS.Mode != nil && *listener.TLS.Mode != v1beta1.TLSModePassthrough { + listener.SetCondition( + v1beta1.ListenerConditionReady, + metav1.ConditionFalse, + "UnsupportedTLSMode", + fmt.Sprintf("TLS %s mode is not supported, TLS mode must be Passthrough.", *listener.TLS.Mode), + ) + break + } + + // With TLS Passthrough, partial wildcards are not allowed in xDS config, so "*", "*w.abc.com" are + // invalid configurations. + // TODO: add regex match to detect partial wildcards like *w.abc.com + if listener.Hostname == nil || *listener.Hostname == "" { + listener.SetCondition( + v1beta1.ListenerConditionReady, + metav1.ConditionFalse, + v1beta1.ListenerReasonInvalid, + "Hostname must not be empty with TLS mode Passthrough.", + ) + break + } + + if len(listener.TLS.CertificateRefs) > 0 { + listener.SetCondition( + v1beta1.ListenerConditionReady, + metav1.ConditionFalse, + v1beta1.ListenerReasonInvalid, + "Listener must not have TLS certificate refs set for TLS mode Passthrough", + ) + break + } } lConditions := listener.GetConditions() @@ -459,31 +543,60 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap continue } + // Add the listener to the Xds IR. servicePort := int32(listener.Port) containerPort := servicePortToContainerPort(servicePort) - // Add the listener to the Xds IR. - irListener := &ir.HTTPListener{ - Name: irListenerName(listener), - Address: "0.0.0.0", - Port: uint32(containerPort), - TLS: irTLSConfig(listener.tlsSecret), - } - if listener.Hostname != nil { - irListener.Hostnames = append(irListener.Hostnames, string(*listener.Hostname)) - } else { - // Hostname specifies the virtual hostname to match for protocol types that define this concept. - // When unspecified, all hostnames are matched. This field is ignored for protocols that don’t require hostname based matching. - // see more https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.Listener. - irListener.Hostnames = append(irListener.Hostnames, "*") + switch listener.Protocol { + case v1beta1.HTTPProtocolType, v1beta1.HTTPSProtocolType: + irListener := &ir.HTTPListener{ + Name: irListenerName(listener), + Address: "0.0.0.0", + Port: uint32(containerPort), + TLS: irTLSConfig(listener.tlsSecret), + } + if listener.Hostname != nil { + irListener.Hostnames = append(irListener.Hostnames, string(*listener.Hostname)) + } else { + // Hostname specifies the virtual hostname to match for protocol types that define this concept. + // When unspecified, all hostnames are matched. This field is ignored for protocols that don’t require hostname based matching. + // see more https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.Listener. + irListener.Hostnames = append(irListener.Hostnames, "*") + } + gwXdsIR.HTTP = append(gwXdsIR.HTTP, irListener) + case v1beta1.TLSProtocolType: + irListener := &ir.TCPListener{ + Name: irListenerName(listener), + Address: "0.0.0.0", + Port: uint32(containerPort), + TLS: &ir.TLSInspectorConfig{ + SNIs: []string{}, + }, + } + if listener.Hostname == nil || *listener.Hostname == "" { + listener.SetCondition( + v1beta1.ListenerConditionReady, + metav1.ConditionFalse, + v1beta1.ListenerReasonInvalid, + "Listener is invalid, see other Conditions for details.", + ) + } + if listener.Hostname != nil && *listener.Hostname != "" { + irListener.TLS.SNIs = append(irListener.TLS.SNIs, string(*listener.Hostname)) + } + gwXdsIR.TCP = append(gwXdsIR.TCP, irListener) } - gwXdsIR.HTTP = append(gwXdsIR.HTTP, irListener) // Add the listener to the Infra IR. Infra IR ports must have a unique port number. if !slices.Contains(foundPorts, servicePort) { foundPorts = append(foundPorts, servicePort) - proto := ir.HTTPProtocolType - if listener.Protocol == v1beta1.HTTPSProtocolType { + var proto ir.ProtocolType + switch listener.Protocol { + case v1beta1.HTTPProtocolType: + proto = ir.HTTPProtocolType + case v1beta1.HTTPSProtocolType: proto = ir.HTTPSProtocolType + case v1beta1.TLSProtocolType: + proto = ir.TLSProtocolType } infraPort := ir.ListenerPort{ Name: string(listener.Name), @@ -525,7 +638,7 @@ func buildRuleRouteDest(backendRef v1beta1.HTTPBackendRef, } if backendRef.Group != nil && *backendRef.Group != "" { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonInvalidKind, @@ -535,7 +648,7 @@ func buildRuleRouteDest(backendRef v1beta1.HTTPBackendRef, } if backendRef.Kind != nil && *backendRef.Kind != KindService { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonInvalidKind, @@ -559,7 +672,7 @@ func buildRuleRouteDest(backendRef v1beta1.HTTPBackendRef, }, resources.ReferenceGrants, ) { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonRefNotPermitted, @@ -570,7 +683,7 @@ func buildRuleRouteDest(backendRef v1beta1.HTTPBackendRef, } if backendRef.Port == nil { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, "PortNotSpecified", @@ -581,7 +694,7 @@ func buildRuleRouteDest(backendRef v1beta1.HTTPBackendRef, service := resources.GetService(NamespaceDerefOr(backendRef.Namespace, httpRoute.Namespace), string(backendRef.Name)) if service == nil { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonBackendNotFound, @@ -599,7 +712,7 @@ func buildRuleRouteDest(backendRef v1beta1.HTTPBackendRef, } if !portFound { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, "PortNotFound", @@ -628,42 +741,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Find out if this route attaches to one of our Gateway's listeners, // and if so, get the list of listeners that allow it to attach for each // parentRef. - var relevantRoute bool - for _, parentRef := range httpRoute.Spec.ParentRefs { - isRelevantParentRef, selectedListeners := GetReferencedListeners(parentRef, gateways) - - // Parent ref is not to a Gateway that we control: skip it - if !isRelevantParentRef { - continue - } - relevantRoute = true - - parentRefCtx := httpRoute.GetRouteParentContext(parentRef) - // Reset conditions since they will be recomputed during translation - parentRefCtx.ResetConditions() - - if !HasReadyListener(selectedListeners) { - parentRefCtx.SetCondition(v1beta1.RouteConditionAccepted, metav1.ConditionFalse, "NoReadyListeners", "There are no ready listeners for this parent ref") - continue - } - - var allowedListeners []*ListenerContext - for _, listener := range selectedListeners { - if listener.AllowsKind(v1beta1.RouteGroupKind{Group: GroupPtr(v1beta1.GroupName), Kind: KindHTTPRoute}) && listener.AllowsNamespace(resources.GetNamespace(httpRoute.Namespace)) { - allowedListeners = append(allowedListeners, listener) - } - } - - if len(allowedListeners) == 0 { - parentRefCtx.SetCondition(v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonNotAllowedByListeners, "No listeners included by this parent ref allowed this attachment.") - continue - } - - parentRefCtx.SetListeners(allowedListeners...) - - parentRefCtx.SetCondition(v1beta1.RouteConditionAccepted, metav1.ConditionTrue, v1beta1.RouteReasonAccepted, "Route is accepted") - } - + relevantRoute := processAllowedListenersForParentRefs(httpRoute, gateways, resources) if !relevantRoute { continue } @@ -672,7 +750,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways for _, parentRef := range httpRoute.parentRefs { // Skip parent refs that did not accept the route - if !parentRef.IsAccepted() { + if !parentRef.IsAccepted(httpRoute) { continue } @@ -700,7 +778,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways case v1beta1.HTTPRouteFilterRequestRedirect: // Can't have two redirects for the same route if redirectResponse != nil { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -722,7 +800,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways redir.Scheme = redirect.Scheme } else { errMsg := fmt.Sprintf("Scheme: %s is unsupported, only 'https' and 'http' are supported", *redirect.Scheme) - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -734,7 +812,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways if redirect.Hostname != nil { if err := isValidHostname(string(*redirect.Hostname)); err != nil { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -763,7 +841,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } default: errMsg := fmt.Sprintf("Redirect path type: %s is invalid, only \"ReplaceFullPath\" and \"ReplacePrefixMatch\" are supported", redirect.Path.Type) - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -780,7 +858,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways redir.StatusCode = &redirectCode } else { errMsg := fmt.Sprintf("Status code %d is invalid, only 302 and 301 are supported", redirectCode) - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -812,7 +890,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways for _, addHeader := range headersToAdd { emptyFilterConfig = false if addHeader.Name == "" { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -823,7 +901,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } // Per Gateway API specification on HTTPHeaderName, : and / are invalid characters in header names if strings.Contains(string(addHeader.Name), "/") || strings.Contains(string(addHeader.Name), ":") { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -842,7 +920,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } if !canAddHeader { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -869,7 +947,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways for _, setHeader := range headersToSet { if setHeader.Name == "" { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -879,7 +957,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } // Per Gateway API specification on HTTPHeaderName, : and / are invalid characters in header names if strings.Contains(string(setHeader.Name), "/") || strings.Contains(string(setHeader.Name), ":") { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -898,7 +976,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } } if !canAddHeader { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -925,7 +1003,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } for _, removedHeader := range headersToRemove { if removedHeader == "" { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -942,7 +1020,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } } if !canRemHeader { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -958,7 +1036,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Update the status if the filter failed to configure any valid headers to add/remove if len(addRequestHeaders) == 0 && len(removeRequestHeaders) == 0 && !emptyFilterConfig { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -969,7 +1047,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // "If a reference to a custom filter type cannot be resolved, the filter MUST NOT be skipped. // Instead, requests that would have been processed by that filter MUST receive a HTTP error response." errMsg := fmt.Sprintf("Unknown custom filter type: %s", filter.Type) - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionResolvedRefs, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, @@ -1028,7 +1106,6 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } for _, backendRef := range rule.BackendRefs { - destination, backendWeight := buildRuleRouteDest(backendRef, parentRef, httpRoute, resources) for _, route := range ruleRoutes { // If the route already has a direct response or redirect configured, then it was from a filter so skip @@ -1043,7 +1120,6 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } } } - } // If the route has no valid backends then just use a direct response and don't fuss with weighted responses @@ -1064,7 +1140,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways var hasHostnameIntersection bool for _, listener := range parentRef.listeners { - hosts := ComputeHosts(httpRoute.Spec.Hostnames, listener.Hostname) + hosts := computeHosts(httpRoute.GetHostnames(), listener.Hostname) if len(hosts) == 0 { continue } @@ -1104,7 +1180,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } irKey := irStringKey(listener.gateway) - irListener := xdsIR[irKey].GetListener(irListenerName(listener)) + irListener := xdsIR[irKey].GetHTTPListener(irListenerName(listener)) if irListener != nil { irListener.Routes = append(irListener.Routes, perHostRoutes...) } @@ -1117,14 +1193,14 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } if !hasHostnameIntersection { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonNoMatchingListenerHostname, "There were no hostname intersections between the HTTPRoute and this parent ref's Listener(s).", ) } else { - parentRef.SetCondition( + parentRef.SetCondition(httpRoute, v1beta1.RouteConditionAccepted, metav1.ConditionTrue, v1beta1.RouteReasonAccepted, @@ -1137,6 +1213,246 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways return relevantHTTPRoutes } +func (t *Translator) ProcessTLSRoutes(tlsRoutes []*v1alpha2.TLSRoute, gateways []*GatewayContext, resources *Resources, xdsIR XdsIRMap) []*TLSRouteContext { + var relevantTLSRoutes []*TLSRouteContext + + for _, t := range tlsRoutes { + if t == nil { + panic("received nil tlsroute") + } + tlsRoute := &TLSRouteContext{TLSRoute: t} + + // Find out if this route attaches to one of our Gateway's listeners, + // and if so, get the list of listeners that allow it to attach for each + // parentRef. + relevantRoute := processAllowedListenersForParentRefs(tlsRoute, gateways, resources) + if !relevantRoute { + continue + } + + relevantTLSRoutes = append(relevantTLSRoutes, tlsRoute) + + for _, parentRef := range tlsRoute.parentRefs { + // Skip parent refs that did not accept the route + if !parentRef.IsAccepted(tlsRoute) { + continue + } + + // Need to compute Route rules within the parentRef loop because + // any conditions that come out of it have to go on each RouteParentStatus, + // not on the Route as a whole. + var routeDestinations []*ir.RouteDestination + + // compute backends + for _, rule := range tlsRoute.Spec.Rules { + for _, backendRef := range rule.BackendRefs { + if backendRef.Group != nil && *backendRef.Group != "" { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionResolvedRefs, + metav1.ConditionFalse, + v1beta1.RouteReasonInvalidKind, + "Group is invalid, only the core API group (specified by omitting the group field or setting it to an empty string) is supported", + ) + continue + } + + if backendRef.Kind != nil && *backendRef.Kind != KindService { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionResolvedRefs, + metav1.ConditionFalse, + v1beta1.RouteReasonInvalidKind, + "Kind is invalid, only Service is supported", + ) + continue + } + + if backendRef.Namespace != nil && string(*backendRef.Namespace) != "" && string(*backendRef.Namespace) != tlsRoute.Namespace { + if !isValidCrossNamespaceRef( + crossNamespaceFrom{ + group: v1beta1.GroupName, + kind: KindTLSRoute, + namespace: tlsRoute.Namespace, + }, + crossNamespaceTo{ + group: "", + kind: KindService, + namespace: string(*backendRef.Namespace), + name: string(backendRef.Name), + }, + resources.ReferenceGrants, + ) { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionResolvedRefs, + metav1.ConditionFalse, + v1beta1.RouteReasonRefNotPermitted, + fmt.Sprintf("Backend ref to service %s/%s not permitted by any ReferenceGrant", *backendRef.Namespace, backendRef.Name), + ) + continue + } + } + + if backendRef.Port == nil { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionResolvedRefs, + metav1.ConditionFalse, + "PortNotSpecified", + "A valid port number corresponding to a port on the Service must be specified", + ) + continue + } + + // TODO: [v1alpha2-v1beta1] Replace with NamespaceDerefOr when TLSRoute graduates to v1beta1. + serviceNamespace := NamespaceDerefOrAlpha(backendRef.Namespace, tlsRoute.Namespace) + service := resources.GetService(serviceNamespace, string(backendRef.Name)) + if service == nil { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionResolvedRefs, + metav1.ConditionFalse, + v1beta1.RouteReasonBackendNotFound, + fmt.Sprintf("Service %s/%s not found", serviceNamespace, string(backendRef.Name)), + ) + continue + } + + var portFound bool + for _, port := range service.Spec.Ports { + if port.Port == int32(*backendRef.Port) { + portFound = true + break + } + } + + if !portFound { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionResolvedRefs, + metav1.ConditionFalse, + "PortNotFound", + fmt.Sprintf("Port %d not found on service %s/%s", *backendRef.Port, serviceNamespace, string(backendRef.Name)), + ) + continue + } + + weight := uint32(1) + if backendRef.Weight != nil { + weight = uint32(*backendRef.Weight) + } + + routeDestinations = append(routeDestinations, &ir.RouteDestination{ + Host: service.Spec.ClusterIP, + Port: uint32(*backendRef.Port), + Weight: weight, + }) + } + + // TODO handle: + // - no valid backend refs + // - sum of weights for valid backend refs is 0 + // - returning 500's for invalid backend refs + // - etc. + } + + var hasHostnameIntersection bool + for _, listener := range parentRef.listeners { + hosts := computeHosts(tlsRoute.GetHostnames(), listener.Hostname) + if len(hosts) == 0 { + continue + } + hasHostnameIntersection = true + + irKey := irStringKey(listener.gateway) + irListener := xdsIR[irKey].GetTCPListener(irListenerName(listener)) + if irListener != nil { + irListener.Destinations = routeDestinations + } + // Theoretically there should only be one parent ref per + // Route that attaches to a given Listener, so fine to just increment here, but we + // might want to check to ensure we're not double-counting. + if len(routeDestinations) > 0 { + listener.IncrementAttachedRoutes() + } + } + + if !hasHostnameIntersection { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionAccepted, + metav1.ConditionFalse, + v1beta1.RouteReasonNoMatchingListenerHostname, + "There were no hostname intersections between the HTTPRoute and this parent ref's Listener(s).", + ) + } else { + parentRef.SetCondition(tlsRoute, + v1beta1.RouteConditionAccepted, + metav1.ConditionTrue, + v1beta1.RouteReasonAccepted, + "Route is accepted", + ) + } + } + } + + return relevantTLSRoutes +} + +// processAllowedListenersForParentRefs finds out if the route attaches to one of our +// Gateways' listeners, and if so, gets the list of listeners that allow it to +// attach for each parentRef. +func processAllowedListenersForParentRefs(routeContext RouteContext, gateways []*GatewayContext, resources *Resources) bool { + var relevantRoute bool + + for _, parentRef := range routeContext.GetParentReferences() { + isRelevantParentRef, selectedListeners := GetReferencedListeners(parentRef, gateways) + + // Parent ref is not to a Gateway that we control: skip it + if !isRelevantParentRef { + continue + } + relevantRoute = true + + parentRefCtx := routeContext.GetRouteParentContext(parentRef) + // Reset conditions since they will be recomputed during translation + parentRefCtx.ResetConditions(routeContext) + + if !HasReadyListener(selectedListeners) { + parentRefCtx.SetCondition(routeContext, + v1beta1.RouteConditionAccepted, + metav1.ConditionFalse, + "NoReadyListeners", + "There are no ready listeners for this parent ref", + ) + continue + } + + var allowedListeners []*ListenerContext + for _, listener := range selectedListeners { + acceptedKind := routeContext.GetRouteType() + if listener.AllowsKind(v1beta1.RouteGroupKind{Group: GroupPtr(v1beta1.GroupName), Kind: v1beta1.Kind(acceptedKind)}) && + listener.AllowsNamespace(resources.GetNamespace(routeContext.GetNamespace())) { + allowedListeners = append(allowedListeners, listener) + } + } + + if len(allowedListeners) == 0 { + parentRefCtx.SetCondition(routeContext, + v1beta1.RouteConditionAccepted, + metav1.ConditionFalse, + v1beta1.RouteReasonNotAllowedByListeners, + "No listeners included by this parent ref allowed this attachment.", + ) + continue + } + + parentRefCtx.SetListeners(allowedListeners...) + + parentRefCtx.SetCondition(routeContext, + v1beta1.RouteConditionAccepted, + metav1.ConditionTrue, + v1beta1.RouteReasonAccepted, + "Route is accepted", + ) + } + return relevantRoute +} + type crossNamespaceFrom struct { group string kind string @@ -1193,7 +1509,6 @@ func isValidCrossNamespaceRef(from crossNamespaceFrom, to crossNamespaceTo, refe // Checks if a hostname is valid according to RFC 1123 and gateway API's requirement that it not be an IP address func isValidHostname(hostname string) error { - if errs := validation.IsDNS1123Subdomain(hostname); errs != nil { return fmt.Errorf("hostname %q is invalid for a redirect filter: %v", hostname, errs) } @@ -1229,8 +1544,8 @@ func irListenerName(listener *ListenerContext) string { return fmt.Sprintf("%s-%s-%s", listener.gateway.Namespace, listener.gateway.Name, listener.Name) } -func routeName(httpRoute *HTTPRouteContext, ruleIdx, matchIdx int) string { - return fmt.Sprintf("%s-%s-rule-%d-match-%d", httpRoute.Namespace, httpRoute.Name, ruleIdx, matchIdx) +func routeName(route RouteContext, ruleIdx, matchIdx int) string { + return fmt.Sprintf("%s-%s-rule-%d-match-%d", route.GetNamespace(), route.GetName(), ruleIdx, matchIdx) } func irTLSConfig(tlsSecret *v1.Secret) *ir.TLSListenerConfig { diff --git a/internal/ir/infra.go b/internal/ir/infra.go index 52e05ea7c39..99d91f103aa 100644 --- a/internal/ir/infra.go +++ b/internal/ir/infra.go @@ -79,6 +79,9 @@ const ( // HTTPSProtocolType accepts HTTP/1.1 or HTTP/2 sessions over TLS. HTTPSProtocolType ProtocolType = "HTTPS" + + // Accepts TLS sessions over TCP. + TLSProtocolType ProtocolType = "TLS" ) // NewInfra returns a new Infra with default parameters. diff --git a/internal/ir/xds.go b/internal/ir/xds.go index d03de20b285..9224762d591 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -8,10 +8,11 @@ import ( ) var ( - ErrHTTPListenerNameEmpty = errors.New("field Name must be specified") - ErrHTTPListenerAddressInvalid = errors.New("field Address must be a valid IP address") - ErrHTTPListenerPortInvalid = errors.New("field Port specified is invalid") + ErrListenerNameEmpty = errors.New("field Name must be specified") + ErrListenerAddressInvalid = errors.New("field Address must be a valid IP address") + ErrListenerPortInvalid = errors.New("field Port specified is invalid") ErrHTTPListenerHostnamesEmpty = errors.New("field Hostnames must be specified with at least a single hostname entry") + ErrTCPListenesSNIsEmpty = errors.New("field SNIs must be specified with at least a single server name entry") ErrTLSServerCertEmpty = errors.New("field ServerCertificate must be specified") ErrTLSPrivateKey = errors.New("field PrivateKey must be specified") ErrHTTPRouteNameEmpty = errors.New("field Name must be specified") @@ -35,6 +36,8 @@ var ( type Xds struct { // HTTP listeners exposed by the gateway. HTTP []*HTTPListener + // TCP Listeners exposed by the gateway. + TCP []*TCPListener } // Validate the fields within the Xds structure. @@ -48,6 +51,24 @@ func (x Xds) Validate() error { return errs } +func (x Xds) GetHTTPListener(name string) *HTTPListener { + for _, listener := range x.HTTP { + if listener.Name == name { + return listener + } + } + return nil +} + +func (x Xds) GetTCPListener(name string) *TCPListener { + for _, listener := range x.TCP { + if listener.Name == name { + return listener + } + } + return nil +} + // HTTPListener holds the listener configuration. // +k8s:deepcopy-gen=true type HTTPListener struct { @@ -68,26 +89,17 @@ type HTTPListener struct { Routes []*HTTPRoute } -func (x Xds) GetListener(name string) *HTTPListener { - for _, listener := range x.HTTP { - if listener.Name == name { - return listener - } - } - return nil -} - // Validate the fields within the HTTPListener structure func (h HTTPListener) Validate() error { var errs error if h.Name == "" { - errs = multierror.Append(errs, ErrHTTPListenerNameEmpty) + errs = multierror.Append(errs, ErrListenerNameEmpty) } if ip := net.ParseIP(h.Address); ip == nil { - errs = multierror.Append(errs, ErrHTTPListenerAddressInvalid) + errs = multierror.Append(errs, ErrListenerAddressInvalid) } if h.Port == 0 { - errs = multierror.Append(errs, ErrHTTPListenerPortInvalid) + errs = multierror.Append(errs, ErrListenerPortInvalid) } if len(h.Hostnames) == 0 { errs = multierror.Append(errs, ErrHTTPListenerHostnamesEmpty) @@ -234,6 +246,20 @@ type RouteDestination struct { Weight uint32 } +// Validate the fields within the RouteDestination structure +func (r RouteDestination) Validate() error { + var errs error + // Only support IP hosts for now + if ip := net.ParseIP(r.Host); ip == nil { + errs = multierror.Append(errs, ErrRouteDestinationHostInvalid) + } + if r.Port == 0 { + errs = multierror.Append(errs, ErrRouteDestinationPortInvalid) + } + + return errs +} + // Add header configures a headder to be added to a request. // +k8s:deepcopy-gen=true type AddHeader struct { @@ -336,20 +362,6 @@ func (r HTTPPathModifier) Validate() error { return errs } -// Validate the fields within the RouteDestination structure -func (r RouteDestination) Validate() error { - var errs error - // Only support IP hosts for now - if ip := net.ParseIP(r.Host); ip == nil { - errs = multierror.Append(errs, ErrRouteDestinationHostInvalid) - } - if r.Port == 0 { - errs = multierror.Append(errs, ErrRouteDestinationPortInvalid) - } - - return errs -} - // StringMatch holds the various match conditions. // Only one of Exact, Prefix or SafeRegex can be set. // +k8s:deepcopy-gen=true @@ -384,3 +396,63 @@ func (s StringMatch) Validate() error { return errs } + +// TCPListener holds the TCP listener configuration. +// +k8s:deepcopy-gen=true +type TCPListener struct { + // Name of the TCPListener + Name string + // Address that the listener should listen on. + Address string + // Port on which the service can be expected to be accessed by clients. + Port uint32 + // TLS information required for TLS Passthrough, If provided, incoming + // connections' server names are inspected and routed to backends accordingly. + TLS *TLSInspectorConfig + // Destinations associated with TCP traffic to the service. + Destinations []*RouteDestination +} + +// Validate the fields within the TCPListener structure +func (h TCPListener) Validate() error { + var errs error + if h.Name == "" { + errs = multierror.Append(errs, ErrListenerNameEmpty) + } + if ip := net.ParseIP(h.Address); ip == nil { + errs = multierror.Append(errs, ErrListenerAddressInvalid) + } + if h.Port == 0 { + errs = multierror.Append(errs, ErrListenerPortInvalid) + } + if h.TLS != nil { + if err := h.TLS.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + for _, route := range h.Destinations { + if err := route.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs +} + +// TLSInspectorConfig holds the configuration required for inspecting TLS +// passthrough connections. +// +k8s:deepcopy-gen=true +type TLSInspectorConfig struct { + // Server names that are compared against the server names of a new connection. + // Wildcard hosts are supported in the prefix form. Partial wildcards are not + // supported, and values like *w.example.com are invalid. + // SNIs are used only in case of TLS Passthrough. + SNIs []string +} + +func (t TLSInspectorConfig) Validate() error { + var errs error + if len(t.SNIs) == 0 { + errs = multierror.Append(errs, ErrTCPListenesSNIsEmpty) + } + return errs +} diff --git a/internal/ir/xds_test.go b/internal/ir/xds_test.go index da45b040cdb..537a01aa0d4 100644 --- a/internal/ir/xds_test.go +++ b/internal/ir/xds_test.go @@ -45,6 +45,34 @@ var ( Routes: []*HTTPRoute{&weightedInvalidBackendsHTTPRoute}, } + // TCPListener + happyTCPListenerTLSPassthrough = TCPListener{ + Name: "happy", + Address: "0.0.0.0", + Port: 80, + TLS: &TLSInspectorConfig{SNIs: []string{"example.com"}}, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + invalidNameTCPListenerTLSPassthrough = TCPListener{ + Address: "0.0.0.0", + Port: 80, + TLS: &TLSInspectorConfig{SNIs: []string{"example.com"}}, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + invalidAddrTCPListenerTLSPassthrough = TCPListener{ + Name: "invalid-addr", + Address: "1.0.0", + Port: 80, + TLS: &TLSInspectorConfig{SNIs: []string{"example.com"}}, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + invalidSNITCPListenerTLSPassthrough = TCPListener{ + Address: "0.0.0.0", + Port: 80, + TLS: &TLSInspectorConfig{SNIs: []string{}}, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + // HTTPRoute happyHTTPRoute = HTTPRoute{ Name: "happy", @@ -244,12 +272,19 @@ func TestValidateXds(t *testing.T) { }, want: nil, }, + { + name: "happy tls", + input: Xds{ + TCP: []*TCPListener{&happyTCPListenerTLSPassthrough}, + }, + want: nil, + }, { name: "invalid listener", input: Xds{ HTTP: []*HTTPListener{&happyHTTPListener, &invalidAddrHTTPListener, &invalidRouteMatchHTTPListener}, }, - want: []error{ErrHTTPListenerAddressInvalid, ErrHTTPRouteMatchEmpty}, + want: []error{ErrListenerAddressInvalid, ErrHTTPRouteMatchEmpty}, }, { name: "invalid backend", @@ -300,12 +335,12 @@ func TestValidateHTTPListener(t *testing.T) { Hostnames: []string{"example.com"}, Routes: []*HTTPRoute{&happyHTTPRoute}, }, - want: []error{ErrHTTPListenerNameEmpty}, + want: []error{ErrListenerNameEmpty}, }, { name: "invalid addr", input: invalidAddrHTTPListener, - want: []error{ErrHTTPListenerAddressInvalid}, + want: []error{ErrListenerAddressInvalid}, }, { name: "invalid port and hostnames", @@ -314,7 +349,7 @@ func TestValidateHTTPListener(t *testing.T) { Address: "1.0.0", Routes: []*HTTPRoute{&happyHTTPRoute}, }, - want: []error{ErrHTTPListenerPortInvalid, ErrHTTPListenerHostnamesEmpty}, + want: []error{ErrListenerPortInvalid, ErrHTTPListenerHostnamesEmpty}, }, { name: "invalid route match", @@ -337,6 +372,48 @@ func TestValidateHTTPListener(t *testing.T) { } } +func TestValidateTCPListener(t *testing.T) { + tests := []struct { + name string + input TCPListener + want []error + }{ + { + name: "tls passthrough happy", + input: happyTCPListenerTLSPassthrough, + want: nil, + }, + { + name: "tls passthrough invalid name", + input: invalidNameTCPListenerTLSPassthrough, + want: []error{ErrListenerNameEmpty}, + }, + { + name: "tls passthrough invalid addr", + input: invalidAddrTCPListenerTLSPassthrough, + want: []error{ErrListenerAddressInvalid}, + }, + { + name: "tls passthrough empty SNIs", + input: invalidSNITCPListenerTLSPassthrough, + want: []error{ErrTCPListenesSNIsEmpty}, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + if test.want == nil { + require.NoError(t, test.input.Validate()) + } else { + got := test.input.Validate() + for _, w := range test.want { + assert.ErrorContains(t, got, w.Error()) + } + } + }) + } +} + func TestValidateTLSListenerConfig(t *testing.T) { tests := []struct { name string @@ -376,7 +453,6 @@ func TestValidateTLSListenerConfig(t *testing.T) { } }) } - } func TestValidateHTTPRoute(t *testing.T) { diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index b765727d610..16c3477e5b9 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -358,6 +358,57 @@ func (in *StringMatch) DeepCopy() *StringMatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TCPListener) DeepCopyInto(out *TCPListener) { + *out = *in + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TLSInspectorConfig) + (*in).DeepCopyInto(*out) + } + if in.Destinations != nil { + in, out := &in.Destinations, &out.Destinations + *out = make([]*RouteDestination, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(RouteDestination) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPListener. +func (in *TCPListener) DeepCopy() *TCPListener { + if in == nil { + return nil + } + out := new(TCPListener) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSInspectorConfig) DeepCopyInto(out *TLSInspectorConfig) { + *out = *in + if in.SNIs != nil { + in, out := &in.SNIs, &out.SNIs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSInspectorConfig. +func (in *TLSInspectorConfig) DeepCopy() *TLSInspectorConfig { + if in == nil { + return nil + } + out := new(TLSInspectorConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSListenerConfig) DeepCopyInto(out *TLSListenerConfig) { *out = *in @@ -397,6 +448,17 @@ func (in *Xds) DeepCopyInto(out *Xds) { } } } + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = make([]*TCPListener, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(TCPListener) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Xds. diff --git a/internal/message/types.go b/internal/message/types.go index 6d7ec1a64d5..c3f5675559b 100644 --- a/internal/message/types.go +++ b/internal/message/types.go @@ -4,6 +4,7 @@ import ( "github.com/telepresenceio/watchable" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/internal/ir" @@ -15,11 +16,13 @@ type ProviderResources struct { GatewayClasses watchable.Map[string, *gwapiv1b1.GatewayClass] Gateways watchable.Map[types.NamespacedName, *gwapiv1b1.Gateway] HTTPRoutes watchable.Map[types.NamespacedName, *gwapiv1b1.HTTPRoute] + TLSRoutes watchable.Map[types.NamespacedName, *gwapiv1a2.TLSRoute] Namespaces watchable.Map[string, *corev1.Namespace] Services watchable.Map[types.NamespacedName, *corev1.Service] GatewayStatuses watchable.Map[types.NamespacedName, *gwapiv1b1.Gateway] HTTPRouteStatuses watchable.Map[types.NamespacedName, *gwapiv1b1.HTTPRoute] + TLSRouteStatuses watchable.Map[types.NamespacedName, *gwapiv1a2.TLSRoute] } func (p *ProviderResources) GetGatewayClasses() []*gwapiv1b1.GatewayClass { @@ -56,6 +59,17 @@ func (p *ProviderResources) GetHTTPRoutes() []*gwapiv1b1.HTTPRoute { return res } +func (p *ProviderResources) GetTLSRoutes() []*gwapiv1a2.TLSRoute { + if p.TLSRoutes.Len() == 0 { + return nil + } + res := make([]*gwapiv1a2.TLSRoute, 0, p.TLSRoutes.Len()) + for _, v := range p.TLSRoutes.LoadAll() { + res = append(res, v) + } + return res +} + func (p *ProviderResources) GetNamespaces() []*corev1.Namespace { if p.Namespaces.Len() == 0 { return nil diff --git a/internal/message/types_test.go b/internal/message/types_test.go index 36e16495471..4b2faa409bd 100644 --- a/internal/message/types_test.go +++ b/internal/message/types_test.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -34,6 +35,12 @@ func TestProviderResources(t *testing.T) { Namespace: "test", }, } + t1 := &gwapiv1a2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tlsroute1", + Namespace: "test", + }, + } s1 := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -46,6 +53,7 @@ func TestProviderResources(t *testing.T) { assert.Nil(t, resources.GetGatewayClasses()) assert.Nil(t, resources.GetGateways()) assert.Nil(t, resources.GetHTTPRoutes()) + assert.Nil(t, resources.GetTLSRoutes()) assert.Nil(t, resources.GetServices()) // Add resources @@ -64,6 +72,12 @@ func TestProviderResources(t *testing.T) { } resources.HTTPRoutes.Store(r1Key, r1) + t1Key := types.NamespacedName{ + Namespace: t1.GetNamespace(), + Name: t1.GetName(), + } + resources.TLSRoutes.Store(t1Key, t1) + s1Key := types.NamespacedName{ Namespace: s1.GetNamespace(), Name: s1.GetName(), @@ -83,6 +97,9 @@ func TestProviderResources(t *testing.T) { hrs := resources.GetHTTPRoutes() assert.Equal(t, len(hrs), 1) + trs := resources.GetTLSRoutes() + assert.Equal(t, len(trs), 1) + svcs := resources.GetServices() assert.Equal(t, len(svcs), 1) @@ -110,6 +127,12 @@ func TestProviderResources(t *testing.T) { Namespace: "test", }, } + t2 := &gwapiv1a2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tlsroute2", + Namespace: "test", + }, + } s2 := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "service2", @@ -131,6 +154,12 @@ func TestProviderResources(t *testing.T) { } resources.HTTPRoutes.Store(r2Key, r2) + t2Key := types.NamespacedName{ + Namespace: t2.GetNamespace(), + Name: t2.GetName(), + } + resources.TLSRoutes.Store(t2Key, t2) + s2Key := types.NamespacedName{ Namespace: s2.GetNamespace(), Name: s2.GetName(), @@ -150,6 +179,9 @@ func TestProviderResources(t *testing.T) { hrs = resources.GetHTTPRoutes() assert.ElementsMatch(t, hrs, []*gwapiv1b1.HTTPRoute{r1, r2}) + trs = resources.GetTLSRoutes() + assert.ElementsMatch(t, trs, []*gwapiv1a2.TLSRoute{t1, t2}) + svcs = resources.GetServices() assert.ElementsMatch(t, svcs, []*corev1.Service{s1, s2}) } diff --git a/internal/provider/kubernetes/config/rbac/role.yaml b/internal/provider/kubernetes/config/rbac/role.yaml index ba7e139e0df..66a48f7f1d4 100644 --- a/internal/provider/kubernetes/config/rbac/role.yaml +++ b/internal/provider/kubernetes/config/rbac/role.yaml @@ -31,6 +31,7 @@ rules: - httproutes - referencegrants - referencepolicies + - tlsroutes verbs: - get - list @@ -42,5 +43,6 @@ rules: - gatewayclasses/status - gateways/status - httproutes/status + - tlsroutes/status verbs: - update diff --git a/internal/provider/kubernetes/helpers.go b/internal/provider/kubernetes/helpers.go new file mode 100644 index 00000000000..8ca309cecf6 --- /dev/null +++ b/internal/provider/kubernetes/helpers.go @@ -0,0 +1,76 @@ +package kubernetes + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// validateParentRefs validates the provided routeParentReferences, returning the +// referenced Gateways managed by Envoy Gateway. The only supported parentRef +// is a Gateway. +func validateParentRefs(ctx context.Context, client client.Client, namespace string, + gatewayClassController gwapiv1b1.GatewayController, + routeParentReferences []gwapiv1b1.ParentReference) ([]gwapiv1b1.Gateway, error) { + + var ret []gwapiv1b1.Gateway + for i := range routeParentReferences { + ref := routeParentReferences[i] + if ref.Kind != nil && *ref.Kind != "Gateway" { + return nil, fmt.Errorf("invalid Kind %q", *ref.Kind) + } + if ref.Group != nil && *ref.Group != gwapiv1b1.GroupName { + return nil, fmt.Errorf("invalid Group %q", *ref.Group) + } + + // Ensure the referenced Gateway exists, using the route's namespace unless + // specified by the parentRef. + ns := namespace + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + gwKey := types.NamespacedName{ + Namespace: ns, + Name: string(ref.Name), + } + + gw := new(gwapiv1b1.Gateway) + if err := client.Get(ctx, gwKey, gw); err != nil { + return nil, fmt.Errorf("failed to get gateway %s/%s: %v", gwKey.Namespace, gwKey.Name, err) + } + + gcKey := types.NamespacedName{Name: string(gw.Spec.GatewayClassName)} + gc := new(gwapiv1b1.GatewayClass) + if err := client.Get(ctx, gcKey, gc); err != nil { + return nil, fmt.Errorf("failed to get gatewayclass %s: %v", gcKey.Name, err) + } + if gc.Spec.ControllerName == gatewayClassController { + ret = append(ret, *gw) + } + } + + return ret, nil +} + +// isRoutePresentInNamespace checks if any kind of Routes - HTTPRoute, TLSRoute +// exists in the namespace ns. +func isRoutePresentInNamespace(ctx context.Context, c client.Client, ns string) (bool, error) { + tlsRouteList := &gwapiv1a2.TLSRouteList{} + if err := c.List(ctx, tlsRouteList, &client.ListOptions{Namespace: ns}); err != nil { + return false, fmt.Errorf("error listing tlsroutes") + } + + httpRouteList := &gwapiv1b1.HTTPRouteList{} + if err := c.List(ctx, httpRouteList, &client.ListOptions{Namespace: ns}); err != nil { + return false, fmt.Errorf("error listing httproutes") + } + + if len(tlsRouteList.Items)+len(httpRouteList.Items) > 0 { + return true, nil + } + return false, nil +} diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go index 6bae34c938f..253e787b318 100644 --- a/internal/provider/kubernetes/httproute.go +++ b/internal/provider/kubernetes/httproute.go @@ -129,7 +129,7 @@ func (r *httpRouteReconciler) getHTTPRoutesForGateway(obj client.Object) []recon requests := []reconcile.Request{} for i := range routes.Items { route := routes.Items[i] - gateways, err := r.validateParentRefs(ctx, &route) + gateways, err := validateParentRefs(ctx, r.client, route.Namespace, r.classController, route.Spec.ParentRefs) if err != nil { r.log.Info("invalid parentRefs for httproute, bypassing reconciliation", "object", obj) continue @@ -195,7 +195,7 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R } // Validate the route. - gws, err := r.validateParentRefs(ctx, &route) + gws, err := validateParentRefs(ctx, r.client, route.Namespace, r.classController, route.Spec.ParentRefs) if err != nil { // Remove the route from the watchable map since it's invalid. r.resources.HTTPRoutes.Delete(routeKey) @@ -274,12 +274,12 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R log.Info("deleted httproute from resource map") // Delete the Namespace and Service from the resource maps if no other - // routes exist in the namespace. - routeList = &gwapiv1b1.HTTPRouteList{} - if err := r.client.List(ctx, routeList, &client.ListOptions{Namespace: request.Namespace}); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing httproutes") + // routes (TLSRoute or HTTPRoute) exist in the namespace. + found, err := isRoutePresentInNamespace(ctx, r.client, request.NamespacedName.Namespace) + if err != nil { + return reconcile.Result{}, err } - if len(routeList.Items) == 0 { + if !found { r.resources.Namespaces.Delete(request.Namespace) log.Info("deleted namespace from resource map") r.resources.Services.Delete(request.NamespacedName) @@ -345,46 +345,3 @@ func (r *httpRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { } r.log.Info("status subscriber shutting down") } - -// validateParentRefs validates parentRefs for the provided route, returning the referenced Gateways -// managed by Envoy Gateway. The only supported parentRef is a Gateway. -func (r *httpRouteReconciler) validateParentRefs(ctx context.Context, route *gwapiv1b1.HTTPRoute) ([]gwapiv1b1.Gateway, error) { - if route == nil { - return nil, fmt.Errorf("httproute is nil") - } - - var ret []gwapiv1b1.Gateway - for i := range route.Spec.ParentRefs { - ref := route.Spec.ParentRefs[i] - if ref.Kind != nil && *ref.Kind != "Gateway" { - return nil, fmt.Errorf("invalid Kind %q", *ref.Kind) - } - if ref.Group != nil && *ref.Group != gwapiv1b1.GroupName { - return nil, fmt.Errorf("invalid Group %q", *ref.Group) - } - // Ensure the referenced Gateway exists, using the route's namespace unless - // specified by the parentRef. - ns := route.Namespace - if ref.Namespace != nil { - ns = string(*ref.Namespace) - } - gwKey := types.NamespacedName{ - Namespace: ns, - Name: string(ref.Name), - } - gw := new(gwapiv1b1.Gateway) - if err := r.client.Get(ctx, gwKey, gw); err != nil { - return nil, fmt.Errorf("failed to get gateway %s/%s: %v", gwKey.Namespace, gwKey.Name, err) - } - gcKey := types.NamespacedName{Name: string(gw.Spec.GatewayClassName)} - gc := new(gwapiv1b1.GatewayClass) - if err := r.client.Get(ctx, gcKey, gc); err != nil { - return nil, fmt.Errorf("failed to get gatewayclass %s: %v", gcKey.Name, err) - } - if gc.Spec.ControllerName == r.classController { - ret = append(ret, *gw) - } - } - - return ret, nil -} diff --git a/internal/provider/kubernetes/httproute_test.go b/internal/provider/kubernetes/httproute_test.go index 6fd509c78b0..c504d312153 100644 --- a/internal/provider/kubernetes/httproute_test.go +++ b/internal/provider/kubernetes/httproute_test.go @@ -732,7 +732,7 @@ func TestValidateParentRefs(t *testing.T) { objs = append(objs, tc.gateways[i]) } r.client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(objs...).Build() - gws, err := r.validateParentRefs(ctx, tc.route) + gws, err := validateParentRefs(ctx, r.client, tc.route.Namespace, r.classController, tc.route.Spec.ParentRefs) if tc.expected { require.NoError(t, err) } else { diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index 1383078572b..127febf5470 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -50,9 +50,13 @@ func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResour if err := newGatewayController(mgr, svr, updateHandler.Writer(), resources); err != nil { return nil, fmt.Errorf("failed to create gateway controller: %w", err) } + if err := newHTTPRouteController(mgr, svr, updateHandler.Writer(), resources); err != nil { return nil, fmt.Errorf("failed to create httproute controller: %w", err) } + if err := newTLSRouteController(mgr, svr, updateHandler.Writer(), resources); err != nil { + return nil, fmt.Errorf("failed to create tlsroute controller: %w", err) + } return &Provider{ manager: mgr, diff --git a/internal/provider/kubernetes/kubernetes_test.go b/internal/provider/kubernetes/kubernetes_test.go index 409f7fa1833..655a0eee7c7 100644 --- a/internal/provider/kubernetes/kubernetes_test.go +++ b/internal/provider/kubernetes/kubernetes_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/api/config/v1alpha1" @@ -62,6 +63,7 @@ func TestProvider(t *testing.T) { "gatewayclass accepted status": testGatewayClassAcceptedStatus, "gateway scheduled status": testGatewayScheduledStatus, "httproute": testHTTPRoute, + "tlsroute": testTLSRoute, } for name, tc := range testcases { t.Run(name, func(t *testing.T) { @@ -83,17 +85,40 @@ func startEnv() (*envtest.Environment, *rest.Config, error) { return env, cfg, nil } -func testGatewayClassController(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { - cli := provider.manager.GetClient() - - gc := &gwapiv1b1.GatewayClass{ +func getGatewayClass(name string) *gwapiv1b1.GatewayClass { + return &gwapiv1b1.GatewayClass{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-gc-controllername", + Name: name, }, Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: v1alpha1.GatewayControllerName, + ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), + }, + } +} + +func getService(name, namespace string, ports map[string]int32) *corev1.Service { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{}, }, } + for name, port := range ports { + service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{ + Name: name, + Port: port, + }) + } + return service +} + +func testGatewayClassController(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { + cli := provider.manager.GetClient() + + gc := getGatewayClass("test-gc-controllername") require.NoError(t, cli.Create(ctx, gc)) defer func() { @@ -109,14 +134,7 @@ func testGatewayClassController(ctx context.Context, t *testing.T, provider *Pro func testGatewayClassAcceptedStatus(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { cli := provider.manager.GetClient() - gc := &gwapiv1b1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-gc-accepted-status", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: v1alpha1.GatewayControllerName, - }, - } + gc := getGatewayClass("test-gc-accepted-status") require.NoError(t, cli.Create(ctx, gc)) defer func() { @@ -145,14 +163,7 @@ func testGatewayClassAcceptedStatus(ctx context.Context, t *testing.T, provider func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { cli := provider.manager.GetClient() - gc := &gwapiv1b1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gc-scheduled-status-test", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - } + gc := getGatewayClass("gc-scheduled-status-test") require.NoError(t, cli.Create(ctx, gc)) // Ensure the GatewayClass reports "Ready". @@ -246,14 +257,7 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { cli := provider.manager.GetClient() - gc := &gwapiv1b1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "httproute-test", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - } + gc := getGatewayClass("httproute-test") require.NoError(t, cli.Create(ctx, gc)) // Ensure the GatewayClass reports ready. @@ -301,24 +305,10 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour require.NoError(t, cli.Delete(ctx, gw)) }() - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: "test", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - }, - { - Name: "https", - Port: 443, - }, - }, - }, - } + svc := getService("test", ns.Name, map[string]int32{ + "http": 80, + "https": 443, + }) require.NoError(t, cli.Create(ctx, svc)) @@ -584,3 +574,123 @@ func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resour }) } } + +func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { + cli := provider.manager.GetClient() + + gc := getGatewayClass("tlsroute-test") + require.NoError(t, cli.Create(ctx, gc)) + + defer func() { + require.NoError(t, cli.Delete(ctx, gc)) + }() + + // Create the namespace for the Gateway under test. + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tlsroute-test"}} + require.NoError(t, cli.Create(ctx, ns)) + + gw := &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tlsroute-test", + Namespace: ns.Name, + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: gwapiv1b1.ObjectName(gc.Name), + Listeners: []gwapiv1b1.Listener{ + { + Name: "test", + Port: gwapiv1b1.PortNumber(int32(8080)), + Protocol: gwapiv1b1.TLSProtocolType, + }, + }, + }, + } + require.NoError(t, cli.Create(ctx, gw)) + + defer func() { + require.NoError(t, cli.Delete(ctx, gw)) + }() + + svc := getService("test", ns.Name, map[string]int32{ + "tls": 90, + }) + + require.NoError(t, cli.Create(ctx, svc)) + + defer func() { + require.NoError(t, cli.Delete(ctx, svc)) + }() + + var testCases = []struct { + name string + route gwapiv1a2.TLSRoute + }{ + { + name: "tlsroute", + route: gwapiv1a2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tlsroute-test", + Namespace: ns.Name, + }, + Spec: gwapiv1a2.TLSRouteSpec{ + CommonRouteSpec: gwapiv1a2.CommonRouteSpec{ + ParentRefs: []gwapiv1a2.ParentReference{ + { + Name: gwapiv1a2.ObjectName(gw.Name), + }, + }, + }, + Hostnames: []gwapiv1a2.Hostname{"test.hostname.local"}, + Rules: []gwapiv1a2.TLSRouteRule{ + { + BackendRefs: []gwapiv1a2.BackendRef{ + { + BackendObjectReference: gwapiv1a2.BackendObjectReference{ + Name: "test", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + require.NoError(t, cli.Create(ctx, &testCase.route)) + defer func() { + require.NoError(t, cli.Delete(ctx, &testCase.route)) + }() + + require.Eventually(t, func() bool { + return resources.TLSRoutes.Len() == 1 + }, defaultWait, defaultTick) + + // Ensure the test TLSRoute in the TLSRoute resources is as expected. + key := types.NamespacedName{ + Namespace: testCase.route.Namespace, + Name: testCase.route.Name, + } + require.Eventually(t, func() bool { + return cli.Get(ctx, key, &testCase.route) == nil + }, defaultWait, defaultTick) + troutes, _ := resources.TLSRoutes.Load(key) + assert.Equal(t, &testCase.route, troutes) + + // Ensure the TLSRoute Namespace is in the Namespace resource map. + require.Eventually(t, func() bool { + _, ok := resources.Namespaces.Load(testCase.route.Namespace) + return ok + }, defaultWait, defaultTick) + + // Ensure the Service is in the resource map. + svcKey := utils.NamespacedName(svc) + require.Eventually(t, func() bool { + _, ok := resources.Services.Load(svcKey) + return ok + }, defaultWait, defaultTick) + }) + } +} diff --git a/internal/provider/kubernetes/rbac.go b/internal/provider/kubernetes/rbac.go index 32a72ee1937..4015f405502 100644 --- a/internal/provider/kubernetes/rbac.go +++ b/internal/provider/kubernetes/rbac.go @@ -1,7 +1,7 @@ package kubernetes -// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=gatewayclasses;gateways;httproutes;referencepolicies;referencegrants,verbs=get;list;watch;update -// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=gatewayclasses/status;gateways/status;httproutes/status,verbs=update +// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=gatewayclasses;gateways;httproutes;tlsroutes;referencepolicies;referencegrants,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=gatewayclasses/status;gateways/status;httproutes/status;tlsroutes/status,verbs=update // RBAC for watched resources of Gateway API controllers. // +kubebuilder:rbac:groups="",resources=secrets;services;namespaces,verbs=get;list;watch diff --git a/internal/provider/kubernetes/testdata/in/tlsroute-experimental-crd.yaml b/internal/provider/kubernetes/testdata/in/tlsroute-experimental-crd.yaml new file mode 100644 index 00000000000..fb30827b003 --- /dev/null +++ b/internal/provider/kubernetes/testdata/in/tlsroute-experimental-crd.yaml @@ -0,0 +1,542 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/1086 + gateway.networking.k8s.io/bundle-version: v0.6.0-dev + gateway.networking.k8s.io/channel: experimental + creationTimestamp: null + name: tlsroutes.gateway.networking.k8s.io +spec: + group: gateway.networking.k8s.io + names: + categories: + - gateway-api + kind: TLSRoute + listKind: TLSRouteList + plural: tlsroutes + singular: tlsroute + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: "The TLSRoute resource is similar to TCPRoute, but can be configured + to match against TLS-specific metadata. This allows more flexibility in + matching streams for a given TLS listener. \n If you need to forward traffic + to a single target for a TLS listener, you could choose to use a TCPRoute + with a TLS listener." + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of TLSRoute. + properties: + hostnames: + description: "Hostnames defines a set of SNI names that should match + against the SNI attribute of TLS ClientHello message in TLS handshake. + This matches the RFC 1123 definition of a hostname with 2 notable + exceptions: \n 1. IPs are not allowed in SNI names per RFC 6066. + 2. A hostname may be prefixed with a wildcard label (`*.`). The + wildcard label must appear by itself as the first label. \n If + a hostname is specified by both the Listener and TLSRoute, there + must be at least one intersecting hostname for the TLSRoute to be + attached to the Listener. For example: \n * A Listener with `test.example.com` + as the hostname matches TLSRoutes that have either not specified + any hostnames, or have specified at least one of `test.example.com` + or `*.example.com`. * A Listener with `*.example.com` as the hostname + matches TLSRoutes that have either not specified any hostnames + or have specified at least one hostname that matches the Listener + hostname. For example, `test.example.com` and `*.example.com` + would both match. On the other hand, `example.com` and `test.example.net` + would not match. \n If both the Listener and TLSRoute have specified + hostnames, any TLSRoute hostnames that do not match the Listener + hostname MUST be ignored. For example, if a Listener specified `*.example.com`, + and the TLSRoute specified `test.example.com` and `test.example.net`, + `test.example.net` must not be considered for a match. \n If both + the Listener and TLSRoute have specified hostnames, and none match + with the criteria above, then the TLSRoute is not accepted. The + implementation must raise an 'Accepted' Condition with a status + of `False` in the corresponding RouteParentStatus. \n Support: Core" + items: + description: "Hostname is the fully qualified domain name of a network + host. This matches the RFC 1123 definition of a hostname with + 2 notable exceptions: \n 1. IPs are not allowed. 2. A hostname + may be prefixed with a wildcard label (`*.`). The wildcard label + must appear by itself as the first label. \n Hostname can be \"precise\" + which is a domain name without the terminating dot of a network + host (e.g. \"foo.example.com\") or \"wildcard\", which is a domain + name prefixed with a single wildcard label (e.g. `*.example.com`). + \n Note that as per RFC1035 and RFC1123, a *label* must consist + of lower case alphanumeric characters or '-', and must start and + end with an alphanumeric character. No other punctuation is allowed." + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + type: array + parentRefs: + description: "ParentRefs references the resources (usually Gateways) + that a Route wants to be attached to. Note that the referenced parent + resource needs to allow this for the attachment to be complete. + For Gateways, that means the Gateway needs to allow attachment from + Routes of this kind and namespace. \n The only kind of parent resource + with \"Core\" support is Gateway. This API may be extended in the + future to support additional kinds of parent resources such as one + of the route kinds. \n It is invalid to reference an identical parent + more than once. It is valid to reference multiple distinct sections + within the same parent resource, such as 2 Listeners within a Gateway. + \n It is possible to separately reference multiple distinct objects + that may be collapsed by an implementation. For example, some implementations + may choose to merge compatible Gateway Listeners together. If that + is the case, the list of routes attached to those resources should + also be merged." + items: + description: "ParentReference identifies an API object (usually + a Gateway) that can be considered a parent of this resource (usually + a route). The only kind of parent resource with \"Core\" support + is Gateway. This API may be extended in the future to support + additional kinds of parent resources, such as HTTPRoute. \n The + API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid." + properties: + group: + default: gateway.networking.k8s.io + description: "Group is the group of the referent. \n Support: + Core" + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: "Kind is kind of the referent. \n Support: Core + (Gateway) \n Support: Custom (Other Resources)" + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: "Name is the name of the referent. \n Support: + Core" + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the namespace of the referent. When + unspecified, this refers to the local namespace of the Route. + \n Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: "Port is the network port this Route targets. It + can be interpreted differently based on the type of parent + resource. \n When the parent resource is a Gateway, this targets + all listeners listening on the specified port that also support + this kind of Route(and select this Route). It's not recommended + to set `Port` unless the networking behaviors specified in + a Route must apply to a specific port as opposed to a listener(s) + whose port(s) may be changed. When both Port and SectionName + are specified, the name and port of the selected listener + must match both specified values. \n Implementations MAY choose + to support other parent resources. Implementations supporting + other types of parent resources MUST clearly document how/if + Port is interpreted. \n For the purpose of status, an attachment + is considered successful as long as the parent resource accepts + it partially. For example, Gateway listeners can restrict + which Routes can attach to them by Route kind, namespace, + or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this + Route, the Route MUST be considered detached from the Gateway. + \n Support: Extended \n " + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: "SectionName is the name of a section within the + target resource. In the following resources, SectionName is + interpreted as the following: \n * Gateway: Listener Name. + When both Port (experimental) and SectionName are specified, + the name and port of the selected listener must match both + specified values. \n Implementations MAY choose to support + attaching Routes to other resources. If that is the case, + they MUST clearly document how SectionName is interpreted. + \n When unspecified (empty string), this will reference the + entire resource. For the purpose of status, an attachment + is considered successful if at least one section in the parent + resource accepts it. For example, Gateway listeners can restrict + which Routes can attach to them by Route kind, namespace, + or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this + Route, the Route MUST be considered detached from the Gateway. + \n Support: Core" + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + rules: + description: Rules are a list of TLS matchers and actions. + items: + description: TLSRouteRule is the configuration for a given rule. + properties: + backendRefs: + description: "BackendRefs defines the backend(s) where matching + requests should be sent. If unspecified or invalid (refers + to a non-existent resource or a Service with no endpoints), + the rule performs no forwarding; if no filters are specified + that would result in a response being sent, the underlying + implementation must actively reject request attempts to this + backend, by rejecting the connection or returning a 500 status + code. Request rejections must respect weight; if an invalid + backend is requested to have 80% of requests, then 80% of + requests must be rejected instead. \n Support: Core for Kubernetes + Service \n Support: Custom for any other resource \n Support + for weight: Extended" + items: + description: "BackendRef defines how a Route should forward + a request to a Kubernetes resource. \n Note that when a + namespace is specified, a ReferenceGrant object is required + in the referent namespace to allow that namespace's owner + to accept the reference. See the ReferenceGrant documentation + for details." + properties: + group: + default: "" + description: Group is the group of the referent. For example, + "networking.k8s.io". When unspecified (empty string), + core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kind is kind of the referent. For example + "HTTPRoute" or "Service". Defaults to "Service" when + not specified. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the namespace of the backend. + When unspecified, the local namespace is inferred. \n + Note that when a namespace is specified, a ReferenceGrant + object is required in the referent namespace to allow + that namespace's owner to accept the reference. See + the ReferenceGrant documentation for details. \n Support: + Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Port specifies the destination port number + to use for this resource. Port is required when the + referent is a Kubernetes Service. In this case, the + port number is the service port number, not the target + port. For other resources, destination port might be + derived from the referent resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: "Weight specifies the proportion of requests + forwarded to the referenced backend. This is computed + as weight/(sum of all weights in this BackendRefs list). + For non-zero values, there may be some epsilon from + the exact proportion defined here depending on the precision + an implementation supports. Weight is not a percentage + and the sum of weights does not need to equal 100. \n + If only one backend is specified and it has a weight + greater than 0, 100% of the traffic is forwarded to + that backend. If weight is set to 0, no traffic should + be forwarded for this entry. If unspecified, weight + defaults to 1. \n Support for this field varies based + on the context where used." + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + maxItems: 16 + minItems: 1 + type: array + type: object + maxItems: 16 + minItems: 1 + type: array + required: + - rules + type: object + status: + description: Status defines the current state of TLSRoute. + properties: + parents: + description: "Parents is a list of parent resources (usually Gateways) + that are associated with the route, and the status of the route + with respect to each parent. When this route attaches to a parent, + the controller that manages the parent must add an entry to this + list when the controller first sees the route and should update + the entry as appropriate when the route or gateway is modified. + \n Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this + API can only populate Route status for the Gateways/parent resources + they are responsible for. \n A maximum of 32 Gateways will be represented + in this list. An empty list means the route has not been attached + to any Gateway." + items: + description: RouteParentStatus describes the status of a route with + respect to an associated Parent. + properties: + conditions: + description: "Conditions describes the status of the route with + respect to the Gateway. Note that the route's availability + is also subject to the Gateway's own status conditions and + listener status. \n If the Route's ParentRef specifies an + existing Gateway that supports Routes of this kind AND that + Gateway's controller has sufficient access, then that Gateway's + controller MUST set the \"Accepted\" condition on the Route, + to indicate whether the route has been accepted or rejected + by the Gateway, and why. \n A Route MUST be considered \"Accepted\" + if at least one of the Route's rules is implemented by the + Gateway. \n There are a number of cases where the \"Accepted\" + condition may not be set due to lack of controller visibility, + that includes when: \n * The Route refers to a non-existent + parent. * The Route is of a type that the controller does + not support. * The Route is in a namespace the controller + does not have access to." + items: + description: "Condition contains details for one aspect of + the current state of this API Resource. --- This struct + is intended for direct use as an array at the field path + .status.conditions. For example, type FooStatus struct{ + \ // Represents the observations of a foo's current state. + \ // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // + +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should + be when the underlying condition changed. If that is + not known, then using the time when the API field changed + is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value should + be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: "ControllerName is a domain/path string that indicates + the name of the controller that wrote this status. This corresponds + with the controllerName field on GatewayClass. \n Example: + \"example.net/gateway-controller\". \n The format of this + field is DOMAIN \"/\" PATH, where DOMAIN and PATH are valid + Kubernetes names (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + \n Controllers MUST populate this field when writing status. + Controllers should ensure that entries to status populated + with their ControllerName are cleaned up when they are no + longer necessary." + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: ParentRef corresponds with a ParentRef in the spec + that this RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: "Group is the group of the referent. \n Support: + Core" + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: "Kind is kind of the referent. \n Support: + Core (Gateway) \n Support: Custom (Other Resources)" + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: "Name is the name of the referent. \n Support: + Core" + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the namespace of the referent. + When unspecified, this refers to the local namespace of + the Route. \n Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: "Port is the network port this Route targets. + It can be interpreted differently based on the type of + parent resource. \n When the parent resource is a Gateway, + this targets all listeners listening on the specified + port that also support this kind of Route(and select this + Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to + a specific port as opposed to a listener(s) whose port(s) + may be changed. When both Port and SectionName are specified, + the name and port of the selected listener must match + both specified values. \n Implementations MAY choose to + support other parent resources. Implementations supporting + other types of parent resources MUST clearly document + how/if Port is interpreted. \n For the purpose of status, + an attachment is considered successful as long as the + parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them + by Route kind, namespace, or hostname. If 1 of 2 Gateway + listeners accept attachment from the referencing Route, + the Route MUST be considered successfully attached. If + no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + \n Support: Extended \n " + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: "SectionName is the name of a section within + the target resource. In the following resources, SectionName + is interpreted as the following: \n * Gateway: Listener + Name. When both Port (experimental) and SectionName are + specified, the name and port of the selected listener + must match both specified values. \n Implementations MAY + choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName + is interpreted. \n When unspecified (empty string), this + will reference the entire resource. For the purpose of + status, an attachment is considered successful if at least + one section in the parent resource accepts it. For example, + Gateway listeners can restrict which Routes can attach + to them by Route kind, namespace, or hostname. If 1 of + 2 Gateway listeners accept attachment from the referencing + Route, the Route MUST be considered successfully attached. + If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + \n Support: Core" + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + required: + - parents + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/internal/provider/kubernetes/tlsroute.go b/internal/provider/kubernetes/tlsroute.go new file mode 100644 index 00000000000..ff4296b53a5 --- /dev/null +++ b/internal/provider/kubernetes/tlsroute.go @@ -0,0 +1,326 @@ +// Portions of this code are based on code from Contour, available at: +// https://github.com/projectcontour/contour/blob/main/internal/controller/tlsroute.go + +package kubernetes + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/message" + "github.com/envoyproxy/gateway/internal/provider/utils" + "github.com/envoyproxy/gateway/internal/status" +) + +const ( + serviceTLSRouteIndex = "serviceTLSRouteBackendRef" +) + +type tlsRouteReconciler struct { + client client.Client + log logr.Logger + statusUpdater status.Updater + classController gwapiv1b1.GatewayController + + resources *message.ProviderResources +} + +// newTLSRouteController creates the tlsroute controller from mgr. The controller will be pre-configured +// to watch for TLSRoute objects across all namespaces. +func newTLSRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources) error { + r := &tlsRouteReconciler{ + client: mgr.GetClient(), + log: cfg.Logger, + classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), + statusUpdater: su, + resources: resources, + } + + c, err := controller.New("tlsroute", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + r.log.Info("created tlsroute controller") + + if err := c.Watch( + &source.Kind{Type: &gwapiv1a2.TLSRoute{}}, + &handler.EnqueueRequestForObject{}, + ); err != nil { + return err + } + + // Subscribe to status updates + go r.subscribeAndUpdateStatus(context.Background()) + + // Add indexing on TLSRoute, for Service objects that are referenced in TLSRoute objects + // via `.spec.rules.backendRefs`. This helps in querying for TLSRoutes that are affected by + // a particular Service CRUD. + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &gwapiv1a2.TLSRoute{}, serviceTLSRouteIndex, func(rawObj client.Object) []string { + tlsRoute := rawObj.(*gwapiv1a2.TLSRoute) + var backendServices []string + for _, rule := range tlsRoute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if string(*backend.Kind) == gatewayapi.KindService { + // If an explicit Service namespace is not provided, use the TLSRoute namespace to + // lookup the provided Service Name. + backendServices = append(backendServices, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOrAlpha(backend.Namespace, tlsRoute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + return backendServices + }); err != nil { + return err + } + + // Watch Gateway CRUDs and reconcile affected TLSRoutes. + if err := c.Watch( + &source.Kind{Type: &gwapiv1b1.Gateway{}}, + handler.EnqueueRequestsFromMapFunc(r.getTLSRoutesForGateway), + ); err != nil { + return err + } + + // Watch Service CRUDs and reconcile affected TLSRoutes. + if err := c.Watch( + &source.Kind{Type: &corev1.Service{}}, + handler.EnqueueRequestsFromMapFunc(r.getTLSRoutesForService), + ); err != nil { + return err + } + + r.log.Info("watching tlsroute objects") + return nil +} + +// getTLSRoutesForGateway uses a Gateway obj to fetch TLSRoutes, iterating +// through them and creating a reconciliation request for each valid TLSRoute +// that references obj. +func (r *tlsRouteReconciler) getTLSRoutesForGateway(obj client.Object) []reconcile.Request { + ctx := context.Background() + + gw, ok := obj.(*gwapiv1b1.Gateway) + if !ok { + r.log.Info("unexpected object type, bypassing reconciliation", "object", obj) + return []reconcile.Request{} + } + + routes := &gwapiv1a2.TLSRouteList{} + if err := r.client.List(ctx, routes); err != nil { + return []reconcile.Request{} + } + + requests := []reconcile.Request{} + for i := range routes.Items { + route := routes.Items[i] + gateways, err := validateParentRefs(ctx, r.client, route.Namespace, r.classController, gatewayapi.UpgradeParentReferences(route.Spec.ParentRefs)) + if err != nil { + r.log.Info("invalid parentRefs for tlsroute, bypassing reconciliation", "object", obj) + continue + } + for j := range gateways { + if gateways[j].Namespace == gw.Namespace && gateways[j].Name == gw.Name { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: route.Namespace, + Name: route.Name, + }, + } + requests = append(requests, req) + break + } + } + } + + return requests +} + +// getTLSRoutesForService uses a Service obj to fetch TLSRoutes that references +// the Service using `.spec.rules.backendRefs`. The affected TLSRoutes are then +// pushed for reconciliation. +func (r *tlsRouteReconciler) getTLSRoutesForService(obj client.Object) []reconcile.Request { + affectedTLSRouteList := &gwapiv1a2.TLSRouteList{} + + if err := r.client.List(context.Background(), affectedTLSRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(serviceTLSRouteIndex, utils.NamespacedName(obj).String()), + }); err != nil { + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, len(affectedTLSRouteList.Items)) + for i, item := range affectedTLSRouteList.Items { + requests[i] = reconcile.Request{ + NamespacedName: utils.NamespacedName(item.DeepCopy()), + } + } + + return requests +} + +func (r *tlsRouteReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := r.log.WithValues("namespace", request.Namespace, "name", request.Name) + + log.Info("reconciling tlsroute") + + // Fetch all TLSRoutes from the cache. + routeList := &gwapiv1a2.TLSRouteList{} + if err := r.client.List(ctx, routeList); err != nil { + return reconcile.Result{}, fmt.Errorf("error listing tlsroutes") + } + + found := false + for i := range routeList.Items { + // See if this route from the list matched the reconciled route. + route := routeList.Items[i] + routeKey := utils.NamespacedName(&route) + if routeKey == request.NamespacedName { + found = true + } + + // Store the tlsroute in the resource map. + r.resources.TLSRoutes.Store(routeKey, &route) + log.Info("added tlsroute to resource map") + + // Get the route's namespace from the cache. + nsKey := types.NamespacedName{Name: route.Namespace} + ns := new(corev1.Namespace) + if err := r.client.Get(ctx, nsKey, ns); err != nil { + if errors.IsNotFound(err) { + // The route's namespace doesn't exist in the cache, so remove it from + // the namespace resource map if it exists. + if _, ok := r.resources.Namespaces.Load(nsKey.Name); ok { + r.resources.Namespaces.Delete(nsKey.Name) + log.Info("deleted namespace from resource map") + } + } + return reconcile.Result{}, fmt.Errorf("failed to get namespace %s", nsKey.Name) + } + + // The route's namespace exists, so add it to the resource map. + r.resources.Namespaces.Store(nsKey.Name, ns) + log.Info("added namespace to resource map") + + // Get the route's backendRefs from the cache. Note that a Service is the + // only supported kind. + for i := range route.Spec.Rules { + for j := range route.Spec.Rules[i].BackendRefs { + ref := route.Spec.Rules[i].BackendRefs[j] + if err := validateTLSRouteBackendRef(&ref); err != nil { + return reconcile.Result{}, fmt.Errorf("invalid backendRef: %w", err) + } + + // The backendRef is valid, so get the referenced service from the cache. + svcKey := types.NamespacedName{Namespace: route.Namespace, Name: string(ref.Name)} + svc := new(corev1.Service) + if err := r.client.Get(ctx, svcKey, svc); err != nil { + if errors.IsNotFound(err) { + // The ref's service doesn't exist in the cache, so remove it from + // the resource map if it exists. + if _, ok := r.resources.Services.Load(svcKey); ok { + r.resources.Services.Delete(svcKey) + log.Info("deleted service from resource map") + } + } + return reconcile.Result{}, fmt.Errorf("failed to get service %s/%s", + svcKey.Namespace, svcKey.Name) + } + + // The backendRef Service exists, so add it to the resource map. + r.resources.Services.Store(svcKey, svc) + log.Info("added service to resource map") + } + } + } + + if !found { + // Delete the tlsroute from the resource map. + r.resources.TLSRoutes.Delete(request.NamespacedName) + log.Info("deleted tlsroute from resource map") + + // Delete the Namespace and Service from the resource maps if no other + // routes (TLSRoute or HTTPRoute) exist in the namespace. + found, err := isRoutePresentInNamespace(ctx, r.client, request.NamespacedName.Namespace) + if err != nil { + return reconcile.Result{}, err + } + if !found { + r.resources.Namespaces.Delete(request.Namespace) + log.Info("deleted namespace from resource map") + r.resources.Services.Delete(request.NamespacedName) + log.Info("deleted service from resource map") + } + } + + log.Info("reconciled tlsroute") + + return reconcile.Result{}, nil +} + +// validateTLSRouteBackendRef validates that ref is a reference to a local Service. +func validateTLSRouteBackendRef(ref *gwapiv1a2.BackendRef) error { + switch { + case ref == nil: + return nil + case ref.Group != nil && *ref.Group != corev1.GroupName: + return fmt.Errorf("invalid group; must be nil or empty string") + case ref.Kind != nil && *ref.Kind != gatewayapi.KindService: + return fmt.Errorf("invalid kind %q; must be %q", + *ref.BackendObjectReference.Kind, gatewayapi.KindService) + case ref.Namespace != nil: + return fmt.Errorf("invalid namespace; must be nil") + } + + return nil +} + +// subscribeAndUpdateStatus subscribes to tlsroute status updates and writes it into the +// Kubernetes API Server +func (r *tlsRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { + // Subscribe to resources + for snapshot := range r.resources.TLSRouteStatuses.Subscribe(ctx) { + r.log.Info("received a status notification") + updates := snapshot.Updates + for _, update := range updates { + // skip delete updates. + if update.Delete { + continue + } + key := update.Key + val := update.Value + r.statusUpdater.Send(status.Update{ + NamespacedName: key, + Resource: new(gwapiv1a2.TLSRoute), + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + t, ok := obj.(*gwapiv1a2.TLSRoute) + if !ok { + panic(fmt.Sprintf("unsupported object type %T", obj)) + } + tCopy := t.DeepCopy() + tCopy.Status.Parents = val.Status.Parents + return tCopy + }), + }) + } + } + r.log.Info("status subscriber shutting down") +} diff --git a/internal/status/status.go b/internal/status/status.go index c58a488c0bc..373a19b3941 100644 --- a/internal/status/status.go +++ b/internal/status/status.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -143,6 +144,7 @@ func (u *UpdateWriter) Send(update Update) { // GatewayClasses // Gateway // HTTPRoute +// TLSRoute func isStatusEqual(objA, objB interface{}) bool { opts := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration") switch a := objA.(type) { @@ -164,6 +166,12 @@ func isStatusEqual(objA, objB interface{}) bool { return true } } + case *gwapiv1a2.TLSRoute: + if b, ok := objB.(*gwapiv1a2.TLSRoute); ok { + if cmp.Equal(a.Status, b.Status, opts) { + return true + } + } } return false } diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index 2a1dff9b309..b98d5dfe806 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -12,18 +12,18 @@ import ( "github.com/envoyproxy/gateway/internal/ir" ) -func buildXdsCluster(httpRoute *ir.HTTPRoute) (*cluster.Cluster, error) { +func buildXdsCluster(routeName string, destinations []*ir.RouteDestination) (*cluster.Cluster, error) { localities := make([]*endpoint.LocalityLbEndpoints, 0, 1) locality := &endpoint.LocalityLbEndpoints{ Locality: &core.Locality{}, - LbEndpoints: buildXdsEndpoints(httpRoute.Destinations), + LbEndpoints: buildXdsEndpoints(destinations), Priority: 0, // Each locality gets the same weight 1. There is a single locality // per priority, so the weight value does not really matter, but some // load balancers need the value to be set. LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}} localities = append(localities, locality) - clusterName := getXdsClusterName(httpRoute.Name) + clusterName := getXdsClusterName(routeName) return &cluster.Cluster{ Name: clusterName, ConnectTimeout: durationpb.New(5 * time.Second), diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index baca1aae39b..e3ea4c82fe8 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -6,7 +6,9 @@ import ( core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" router "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + tls_inspector "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/tls_inspector/v3" hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + tcp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" tls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" "google.golang.org/protobuf/types/known/anypb" @@ -71,6 +73,74 @@ func buildXdsListener(httpListener *ir.HTTPListener) (*listener.Listener, error) }, nil } +func buildXdsTCPListener(clusterName string, tcpListener *ir.TCPListener) (*listener.Listener, error) { + if tcpListener == nil { + return nil, errors.New("http listener is nil") + } + + statPrefix := "tcp" + if tcpListener.TLS != nil { + statPrefix = "passthrough" + } + mgr := &tcp.TcpProxy{ + StatPrefix: statPrefix, + ClusterSpecifier: &tcp.TcpProxy_Cluster{ + Cluster: clusterName, + }, + } + mgrAny, err := anypb.New(mgr) + if err != nil { + return nil, err + } + + filterChain := &listener.FilterChain{ + Filters: []*listener.Filter{{ + Name: wellknown.TCPProxy, + ConfigType: &listener.Filter_TypedConfig{ + TypedConfig: mgrAny, + }, + }}, + } + if tcpListener.TLS != nil { + filterChain.FilterChainMatch = &listener.FilterChainMatch{ + ServerNames: tcpListener.TLS.SNIs, + } + } + + xdsListener := &listener.Listener{ + Name: getXdsListenerName(tcpListener.Name, tcpListener.Port), + Address: &core.Address{ + Address: &core.Address_SocketAddress{ + SocketAddress: &core.SocketAddress{ + Protocol: core.SocketAddress_TCP, + Address: tcpListener.Address, + PortSpecifier: &core.SocketAddress_PortValue{ + PortValue: tcpListener.Port, + }, + }, + }, + }, + FilterChains: []*listener.FilterChain{filterChain}, + } + + if tcpListener.TLS != nil { + tlsInspector := &tls_inspector.TlsInspector{} + tlsInspectorAny, err := anypb.New(tlsInspector) + if err != nil { + return nil, err + } + + xdsListener.ListenerFilters = []*listener.ListenerFilter{{ + Name: wellknown.TlsInspector, + ConfigType: &listener.ListenerFilter_TypedConfig{ + TypedConfig: tlsInspectorAny, + }, + }} + } + + return xdsListener, nil +} + func buildXdsDownstreamTLSSocket(listenerName string, tlsConfig *ir.TLSListenerConfig) (*core.TransportSocket, error) { tlsCtx := &tls.DownstreamTlsContext{ @@ -113,5 +183,4 @@ func buildXdsDownstreamTLSSecret(listenerName string, }, }, }, nil - } diff --git a/internal/xds/translator/testdata/in/xds-ir/tls-route-passthrough.yaml b/internal/xds/translator/testdata/in/xds-ir/tls-route-passthrough.yaml new file mode 100644 index 00000000000..c7f59633067 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/tls-route-passthrough.yaml @@ -0,0 +1,12 @@ +tcp: +- name: "tls-passthrough" + address: "0.0.0.0" + port: 10080 + tls: + snis: + - foo.com + destinations: + - host: "1.2.3.4" + port: 50000 + - host: "5.6.7.8" + port: 50001 diff --git a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml new file mode 100644 index 00000000000..95ed98685be --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml @@ -0,0 +1,23 @@ +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: cluster_tls-passthrough + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + - endpoint: + address: + socketAddress: + address: 5.6.7.8 + portValue: 50001 + loadBalancingWeight: 1 + locality: {} + name: cluster_tls-passthrough + outlierDetection: {} + type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml new file mode 100644 index 00000000000..b52cb5d8d2b --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml @@ -0,0 +1,19 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + filterChains: + - filterChainMatch: + serverNames: + - foo.com + filters: + - name: envoy.filters.network.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: cluster_tls-passthrough + statPrefix: passthrough + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + name: listener_tls-passthrough_10080 diff --git a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.routes.yaml new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.routes.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 3c6bf26147c..434b0e62e8b 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -64,7 +64,7 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { if len(httpRoute.Destinations) == 0 && httpRoute.BackendWeights.Invalid > 0 { continue } - xdsCluster, err := buildXdsCluster(httpRoute) + xdsCluster, err := buildXdsCluster(httpRoute.Name, httpRoute.Destinations) if err != nil { return nil, multierror.Append(err, errors.New("error building xds cluster")) } @@ -81,6 +81,22 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { tCtx.AddXdsResource(resource.RouteType, xdsRouteCfg) } + for _, tcpListener := range ir.TCP { + // 1:1 between IR TCPListener and xDS Cluster + xdsCluster, err := buildXdsCluster(tcpListener.Name, tcpListener.Destinations) + if err != nil { + return nil, multierror.Append(err, errors.New("error building xds cluster")) + } + tCtx.AddXdsResource(resource.ClusterType, xdsCluster) + + // 1:1 between IR TCPListener and xDS Listener + xdsListener, err := buildXdsTCPListener(xdsCluster.Name, tcpListener) + if err != nil { + return nil, multierror.Append(err, errors.New("error building xds listener")) + } + + tCtx.AddXdsResource(resource.ListenerType, xdsListener) + } return tCtx, nil } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 90cea496704..0102b6d84f7 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -50,6 +50,9 @@ func TestTranslate(t *testing.T) { name: "simple-tls", requireSecrets: true, }, + { + name: "tls-route-passthrough", + }, } for _, tc := range testCases { From 4d8d9f65c1a569ed941491933cbba77831486cf6 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Wed, 12 Oct 2022 02:48:35 +0800 Subject: [PATCH 009/113] feat: implement liveness and readiness probes (#524) * feat: implement liveness and readiness probes Signed-off-by: bitliu * fix: remove duplicate case Signed-off-by: bitliu * Fix merge conflicts Signed-off-by: Arko Dasgupta Signed-off-by: bitliu Signed-off-by: Arko Dasgupta Co-authored-by: Arko Dasgupta --- .../config/envoy-gateway/deploy_and_ns.yaml | 12 ++++++++++ internal/provider/kubernetes/kubernetes.go | 23 +++++++++++++++---- test/conformance/conformance_test.go | 1 - 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/internal/provider/kubernetes/config/envoy-gateway/deploy_and_ns.yaml b/internal/provider/kubernetes/config/envoy-gateway/deploy_and_ns.yaml index f16cdb1e8be..1b0b45c2482 100644 --- a/internal/provider/kubernetes/config/envoy-gateway/deploy_and_ns.yaml +++ b/internal/provider/kubernetes/config/envoy-gateway/deploy_and_ns.yaml @@ -42,6 +42,18 @@ spec: - name: certs mountPath: /certs readOnly: true + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 # TODO(user): Configure the resources accordingly based on the project requirements. # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index 127febf5470..9c265153329 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -7,6 +7,7 @@ import ( "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/envoyproxy/gateway/internal/envoygateway" @@ -27,11 +28,12 @@ type Provider struct { func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResources) (*Provider, error) { // TODO: Decide which mgr opts should be exposed through envoygateway.provider.kubernetes API. mgrOpts := manager.Options{ - Scheme: envoygateway.GetScheme(), - Logger: svr.Logger, - LeaderElection: false, - LeaderElectionID: "5b9825d2.gateway.envoyproxy.io", - MetricsBindAddress: ":8080", + Scheme: envoygateway.GetScheme(), + Logger: svr.Logger, + LeaderElection: false, + HealthProbeBindAddress: ":8081", + LeaderElectionID: "5b9825d2.gateway.envoyproxy.io", + MetricsBindAddress: ":8080", } mgr, err := ctrl.NewManager(cfg, mgrOpts) if err != nil { @@ -54,10 +56,21 @@ func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResour if err := newHTTPRouteController(mgr, svr, updateHandler.Writer(), resources); err != nil { return nil, fmt.Errorf("failed to create httproute controller: %w", err) } + if err := newTLSRouteController(mgr, svr, updateHandler.Writer(), resources); err != nil { return nil, fmt.Errorf("failed to create tlsroute controller: %w", err) } + // Add health check health probes. + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return nil, fmt.Errorf("unable to set up health check: %w", err) + } + + // Add ready check health probes. + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return nil, fmt.Errorf("unable to set up ready check: %w", err) + } + return &Provider{ manager: mgr, client: mgr.GetClient(), diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index 6273fd69e1c..e69f5586b71 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -50,7 +50,6 @@ func TestGatewayAPIConformance(t *testing.T) { tests.HTTPRouteInvalidNonExistentBackendRef, tests.HTTPRouteInvalidBackendRefUnknownKind, tests.HTTPRouteInvalidCrossNamespaceBackendRef, - tests.HTTPRouteHeaderMatching, } cSuite.Run(t, egTests) From 3ed86dfcd24e905609d39722297a25e3246db2e5 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Tue, 11 Oct 2022 13:18:00 -0700 Subject: [PATCH 010/113] Adds TLS Termination Support (#519) * Adds Support for TLS Termination Signed-off-by: danehans * Resolves @arkodg Feedback Signed-off-by: danehans * Resolves @arkodg 10-11-22 Feedback Signed-off-by: danehans Signed-off-by: danehans --- internal/cmd/server.go | 2 + internal/envoygateway/scheme.go | 4 +- internal/gatewayapi/helpers.go | 4 + internal/gatewayapi/runner/runner.go | 8 +- internal/message/types.go | 25 ++ internal/provider/kubernetes/gateway.go | 277 ++++++++++++ internal/provider/kubernetes/gateway_test.go | 422 ++++++++++++++++++ .../in/referencegrant-experimental-crd.yaml | 149 +++++++ test/conformance/conformance_test.go | 7 + 9 files changed, 894 insertions(+), 4 deletions(-) create mode 100644 internal/provider/kubernetes/testdata/in/referencegrant-experimental-crd.yaml diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 052d0229e49..90e2c28cbd9 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -150,6 +150,8 @@ func setupRunners(cfg *config.Server) error { pResources.Gateways.Close() pResources.HTTPRoutes.Close() pResources.Services.Close() + pResources.Secrets.Close() + pResources.ReferenceGrants.Close() pResources.Namespaces.Close() pResources.GatewayStatuses.Close() pResources.HTTPRouteStatuses.Close() diff --git a/internal/envoygateway/scheme.go b/internal/envoygateway/scheme.go index 31fc950c4cd..bc7d5775534 100644 --- a/internal/envoygateway/scheme.go +++ b/internal/envoygateway/scheme.go @@ -3,8 +3,8 @@ package envoygateway import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1alpha2" - gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1beta1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/api/config/v1alpha1" ) diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index a9c2a93a4d4..70464b3e88a 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -32,6 +32,10 @@ func SectionNamePtr(name string) *v1beta1.SectionName { return §ionName } +func TLSModeTypePtr(mode v1beta1.TLSModeType) *v1beta1.TLSModeType { + return &mode +} + func StringPtr(val string) *string { return &val } diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index d4793d5ecdb..a45ee195ec7 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -43,6 +43,8 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Subscribe to resources gatewayClassesCh := r.ProviderResources.GatewayClasses.Subscribe(ctx) gatewaysCh := r.ProviderResources.Gateways.Subscribe(ctx) + secretsCh := r.ProviderResources.Secrets.Subscribe(ctx) + refGrantsCh := r.ProviderResources.ReferenceGrants.Subscribe(ctx) httpRoutesCh := r.ProviderResources.HTTPRoutes.Subscribe(ctx) tlsRoutesCh := r.ProviderResources.TLSRoutes.Subscribe(ctx) servicesCh := r.ProviderResources.Services.Subscribe(ctx) @@ -54,6 +56,8 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { select { case <-gatewayClassesCh: case <-gatewaysCh: + case <-secretsCh: + case <-refGrantsCh: case <-httpRoutesCh: case <-tlsRoutesCh: case <-servicesCh: @@ -62,6 +66,8 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { r.Logger.Info("received a notification") // Load all resources required for translation in.Gateways = r.ProviderResources.GetGateways() + in.Secrets = r.ProviderResources.GetSecrets() + in.ReferenceGrants = r.ProviderResources.GetReferenceGrants() in.HTTPRoutes = r.ProviderResources.GetHTTPRoutes() in.TLSRoutes = r.ProviderResources.GetTLSRoutes() in.Services = r.ProviderResources.GetServices() @@ -81,8 +87,6 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Translate to IR result := t.Translate(&in) - yamlXdsIR, _ := yaml.Marshal(&result.XdsIR) - r.Logger.WithValues("output", "xds-ir").Info(string(yamlXdsIR)) yamlInfraIR, _ := yaml.Marshal(&result.InfraIR) r.Logger.WithValues("output", "infra-ir").Info(string(yamlInfraIR)) diff --git a/internal/message/types.go b/internal/message/types.go index c3f5675559b..33861192477 100644 --- a/internal/message/types.go +++ b/internal/message/types.go @@ -19,6 +19,9 @@ type ProviderResources struct { TLSRoutes watchable.Map[types.NamespacedName, *gwapiv1a2.TLSRoute] Namespaces watchable.Map[string, *corev1.Namespace] Services watchable.Map[types.NamespacedName, *corev1.Service] + Secrets watchable.Map[types.NamespacedName, *corev1.Secret] + + ReferenceGrants watchable.Map[types.NamespacedName, *gwapiv1a2.ReferenceGrant] GatewayStatuses watchable.Map[types.NamespacedName, *gwapiv1b1.Gateway] HTTPRouteStatuses watchable.Map[types.NamespacedName, *gwapiv1b1.HTTPRoute] @@ -93,6 +96,28 @@ func (p *ProviderResources) GetServices() []*corev1.Service { return res } +func (p *ProviderResources) GetSecrets() []*corev1.Secret { + if p.Secrets.Len() == 0 { + return nil + } + res := make([]*corev1.Secret, 0, p.Secrets.Len()) + for _, v := range p.Secrets.LoadAll() { + res = append(res, v) + } + return res +} + +func (p *ProviderResources) GetReferenceGrants() []*gwapiv1a2.ReferenceGrant { + if p.ReferenceGrants.Len() == 0 { + return nil + } + res := make([]*gwapiv1a2.ReferenceGrant, 0, p.ReferenceGrants.Len()) + for _, v := range p.ReferenceGrants.LoadAll() { + res = append(res, v) + } + return res +} + // XdsIR message type XdsIR struct { watchable.Map[string, *ir.Xds] diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index 7a2a9842604..4bd2ba007dc 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/internal/envoygateway/config" @@ -80,6 +81,16 @@ func newGatewayController(mgr manager.Manager, cfg *config.Server, su status.Upd if err := c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, r.enqueueRequestForOwningGateway()); err != nil { return err } + // Trigger gateway reconciliation when a Secret that is referenced + // by a managed Gateway has changed. + if err := c.Watch(&source.Kind{Type: &corev1.Secret{}}, r.enqueueRequestForGatewaySecrets()); err != nil { + return err + } + // Trigger gateway reconciliation when a ReferenceGrant that refers + // to a managed Gateway has changed. + if err := c.Watch(&source.Kind{Type: &gwapiv1a2.ReferenceGrant{}}, r.enqueueRequestForReferencedGateway()); err != nil { + return err + } return nil } @@ -136,6 +147,106 @@ func (r *gatewayReconciler) enqueueRequestForOwningGateway() handler.EventHandle }) } +// enqueueRequestForGatewaySecrets returns an event handler that maps events for +// Secrets referenced by managed Gateways to reconcile requests for those Gateway objects. +func (r *gatewayReconciler) enqueueRequestForGatewaySecrets() handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + secret, ok := a.(*corev1.Secret) + if !ok { + r.log.Info("bypassing reconciliation due to unexpected object type", "type", a) + return nil + } + + ctx := context.Background() + var gateways gwapiv1b1.GatewayList + if err := r.client.List(ctx, &gateways); err != nil { + return nil + } + + var reqs []reconcile.Request + for i := range gateways.Items { + gw := gateways.Items[i] + if r.hasMatchingController(&gw) { + for j := range gw.Spec.Listeners { + if terminatesTLS(&gw.Spec.Listeners[j]) { + secrets, _, err := r.secretsAndRefGrantsForGateway(ctx, &gw) + if err != nil { + return nil + } + for _, s := range secrets { + if s.Namespace == secret.Namespace && s.Name == secret.Name { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: gw.Namespace, + Name: gw.Name, + }, + } + reqs = append(reqs, req) + } + } + } + } + } + } + + return reqs + }) +} + +// enqueueRequestForReferencedGateway returns an event handler that maps events for +// resources that reference a managed Gateway to reconcile requests for those Gateway objects. +// Note: A ReferenceGrant is the only supported object type. +func (r *gatewayReconciler) enqueueRequestForReferencedGateway() handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + rg, ok := a.(*gwapiv1a2.ReferenceGrant) + if !ok { + r.log.Info("bypassing reconciliation due to unexpected object type", "type", a) + return nil + } + + var refs []types.NamespacedName + for _, to := range rg.Spec.To { + if to.Group == gwapiv1a2.GroupName && + to.Kind == gatewayapi.KindGateway && + to.Name != nil { + ref := types.NamespacedName{Namespace: rg.Namespace, Name: string(*to.Name)} + refs = append(refs, ref) + } + } + for _, from := range rg.Spec.From { + if from.Group == gwapiv1a2.GroupName && + from.Kind == gatewayapi.KindGateway { + ref := types.NamespacedName{Namespace: string(from.Namespace), Name: rg.Name} + refs = append(refs, ref) + } + } + + ctx := context.Background() + var gateways gwapiv1b1.GatewayList + if err := r.client.List(ctx, &gateways); err != nil { + return nil + } + + var reqs []reconcile.Request + for i := range gateways.Items { + gw := gateways.Items[i] + for _, ref := range refs { + if gw.Namespace == ref.Namespace && gw.Name == ref.Name && r.hasMatchingController(&gw) { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: gw.Namespace, + Name: gw.Name, + }, + } + reqs = append(reqs, req) + } + } + } + + return reqs + }) +} + // Reconcile finds all the Gateways for the GatewayClass with an "Accepted: true" condition // and passes all Gateways for the configured GatewayClass to the IR for processing. func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -179,6 +290,7 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req } found := false + var secrets []corev1.Secret // Set status conditions for all accepted gateways. for i := range acceptedGateways { gw := acceptedGateways[i] @@ -197,6 +309,37 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req "namespace", gw.Namespace, "name", gw.Name) } + // Get the secret and referenceGrants of the Gateway's TLS configuration. + secrets, refGrants, err := r.secretsAndRefGrantsForGateway(ctx, &gw) + if err != nil { + r.log.Info("failed to get secrets and referencegrants for gateway", + "namespace", gw.Namespace, "name", gw.Name) + } + for i := range secrets { + secret := secrets[i] + // Store the secrets in the resource map. + key := utils.NamespacedName(&secret) + r.resources.Secrets.Store(key, &secret) + } + for i := range refGrants { + rg := refGrants[i] + // Store the referencegrants in the resource map. + key := utils.NamespacedName(&rg) + r.resources.ReferenceGrants.Store(key, &rg) + // Store the referencegrant namespace in the resource map. + key = types.NamespacedName{Name: rg.Namespace} + refNs := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: rg.Namespace, + }, + } + if err := r.client.Get(ctx, key, &refNs); err != nil { + r.log.Info("failed to get referencegrant namespace", "name", refNs.Name) + return reconcile.Result{}, nil + } + r.resources.Namespaces.Store(refNs.Name, &refNs) + } + // update scheduled condition status.UpdateGatewayStatusScheduledCondition(&gw, true) // update address field and ready condition @@ -234,6 +377,24 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req if !found { r.resources.Gateways.Delete(request.NamespacedName) + // Delete the TLS secrets from the resource map if no other managed + // Gateways reference them. + for i := range secrets { + secret := secrets[i] + referenced, err := r.gatewaysRefSecret(ctx, &secret) + switch { + case err != nil: + r.log.Error(err, "failed to verify if other gateways reference secret") + case referenced: + r.log.Info("no other gateways reference secret; deleting from resource map", + "namespace", secret.Namespace, "name", secret.Name) + key := utils.NamespacedName(&secret) + r.resources.Secrets.Delete(key) + default: + r.log.Info("no other gateways reference secret; deleting from resource map", + "namespace", secret.Namespace, "name", secret.Name) + } + } } r.log.WithName(request.Namespace).WithName(request.Name).Info("reconciled gateway") @@ -301,6 +462,122 @@ func (r *gatewayReconciler) envoyServiceForGateway(ctx context.Context, gateway return svc, nil } +// gatewaysRefSecret returns true if a managed Gateway references the provided secret. +// An error is returned if an error is encountered while checking. +func (r *gatewayReconciler) gatewaysRefSecret(ctx context.Context, secret *corev1.Secret) (bool, error) { + if secret == nil { + return false, fmt.Errorf("secret is nil") + } + gateways := &gwapiv1b1.GatewayList{} + if err := r.client.List(ctx, gateways); err != nil { + return false, fmt.Errorf("error listing gatewayclasses: %v", err) + } + for i := range gateways.Items { + gw := gateways.Items[i] + if r.hasMatchingController(&gw) { + secrets, _, err := r.secretsAndRefGrantsForGateway(ctx, &gw) + if err != nil { + return false, err + } + for _, s := range secrets { + if s.Namespace == secret.Namespace && s.Name == secret.Name { + return true, nil + } + } + } + } + + return false, nil +} + +// secretsAndRefGrantsForGateway returns the Secrets referenced by the provided gateway listeners. +// If the provided Gateway references a Secret in a different namespace, a list of +// ReferenceGrants is returned that permit the cross namespace Secret reference. +func (r *gatewayReconciler) secretsAndRefGrantsForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) ([]corev1.Secret, []gwapiv1a2.ReferenceGrant, error) { + var secrets []corev1.Secret + var returnedGrants []gwapiv1a2.ReferenceGrant + for i := range gateway.Spec.Listeners { + listener := gateway.Spec.Listeners[i] + if terminatesTLS(&listener) { + for j := range listener.TLS.CertificateRefs { + ref := listener.TLS.CertificateRefs[j] + if refsSecret(&ref) { + if ref.Namespace != nil { + // A ReferenceGrant is required for cross namespace secret references. + refGrants := &gwapiv1a2.ReferenceGrantList{} + opts := client.ListOptions{Namespace: string(*ref.Namespace)} + if err := r.client.List(ctx, refGrants, &opts); err != nil { + return nil, nil, fmt.Errorf("error listing referencegrants") + } + var gwRefd, secretRefd bool + for _, rg := range refGrants.Items { + for _, from := range rg.Spec.From { + if from.Group == gwapiv1a2.GroupName && + from.Kind == gatewayapi.KindGateway { + gwRefd = true + break + } + } + for _, to := range rg.Spec.To { + if to.Group == corev1.GroupName && + to.Kind == gatewayapi.KindSecret { + if to.Name == nil || *to.Name == gwapiv1a2.ObjectName(ref.Name) { + secretRefd = true + break + } + } + } + if gwRefd && secretRefd { + returnedGrants = append(returnedGrants, rg) + key := types.NamespacedName{ + Namespace: string(*ref.Namespace), + Name: string(ref.Name), + } + secret := new(corev1.Secret) + if err := r.client.Get(ctx, key, secret); err != nil { + return nil, nil, fmt.Errorf("failed to get secret: %v", err) + } + secrets = append(secrets, *secret) + } + } + } else { + // The secret is in the Gateway's namespace. + key := types.NamespacedName{ + Namespace: gateway.Namespace, + Name: string(ref.Name), + } + secret := new(corev1.Secret) + if err := r.client.Get(ctx, key, secret); err != nil { + return nil, nil, fmt.Errorf("failed to get secret: %v", err) + } + secrets = append(secrets, *secret) + } + } + } + } + } + + return secrets, returnedGrants, nil +} + +// terminatesTLS returns true if the provided gateway contains a listener configured +// for TLS termination. +func terminatesTLS(listener *gwapiv1b1.Listener) bool { + if listener.TLS != nil && + listener.Protocol == gwapiv1b1.HTTPSProtocolType && + listener.TLS.Mode != nil && + *listener.TLS.Mode == gwapiv1b1.TLSModeTerminate { + return true + } + return false +} + +// refsSecret returns true if ref refers to a Secret. +func refsSecret(ref *gwapiv1b1.SecretObjectReference) bool { + return (ref.Group == nil || *ref.Group == corev1.GroupName) && + (ref.Kind == nil || *ref.Kind == gatewayapi.KindSecret) +} + // addFinalizer adds the gatewayclass finalizer to the provided gc, if it doesn't exist. func (r *gatewayReconciler) addFinalizer(ctx context.Context, gc *gwapiv1b1.GatewayClass) error { if !slice.ContainsString(gc.Finalizers, gatewayClassFinalizer) { diff --git a/internal/provider/kubernetes/gateway_test.go b/internal/provider/kubernetes/gateway_test.go index 4138364e807..2240f3692b6 100644 --- a/internal/provider/kubernetes/gateway_test.go +++ b/internal/provider/kubernetes/gateway_test.go @@ -6,14 +6,17 @@ import ( "testing" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/log" ) @@ -419,3 +422,422 @@ func TestRemoveFinalizer(t *testing.T) { }) } } + +func TestSecretsAndRefGrantsForGateway(t *testing.T) { + testCases := []struct { + name string + gw *gwapiv1b1.Gateway + secrets []corev1.Secret + refGrants []gwapiv1a2.ReferenceGrant + }{ + { + name: "gateway with no https listeners", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "http", + Port: gwapiv1b1.PortNumber(int32(80)), + Protocol: gwapiv1b1.HTTPProtocolType, + }, + }, + }, + }, + }, + { + name: "gateway with one https listener and one same namespace secret", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "tls", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret"), + }, + }, + }, + }, + }, + }, + }, + secrets: []corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + }, + }, + { + name: "gateway with one http and one https listener with one same namespace secret", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "http", + Port: gwapiv1b1.PortNumber(int32(80)), + Protocol: gwapiv1b1.HTTPProtocolType, + }, + { + Name: "tls", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret"), + }, + }, + }, + }, + }, + }, + }, + secrets: []corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + }, + }, + { + name: "gateway with two https listeners each with two same namespace secrets", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "tls1", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret1"), + }, + { + Name: gwapiv1b1.ObjectName("test-secret2"), + }, + }, + }, + }, + { + Name: "tls2", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret3"), + }, + { + Name: gwapiv1b1.ObjectName("test-secret4"), + }, + }, + }, + }, + }, + }, + }, + secrets: []corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret1", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret2", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret3", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret4", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + }, + }, + { + name: "gateway with one https listener and two same namespace secret", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "tls", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret1"), + }, + { + Name: gwapiv1b1.ObjectName("test-secret2"), + }, + }, + }, + }, + }, + }, + }, + secrets: []corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret1", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret2", + Namespace: "test-ns", + ResourceVersion: "1", + }, + }, + }, + }, + { + name: "gateway with one https listener and one different namespace secret", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "tls", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret"), + Namespace: gatewayapi.NamespacePtr("test-ns2"), + }, + }, + }, + }, + }, + }, + }, + secrets: []corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-ns2", + ResourceVersion: "1", + }, + }, + }, + refGrants: []gwapiv1a2.ReferenceGrant{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "ReferenceGrant", + APIVersion: gwapiv1a2.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-refgrant", + Namespace: "test-ns2", + }, + Spec: gwapiv1a2.ReferenceGrantSpec{ + From: []gwapiv1a2.ReferenceGrantFrom{ + { + Group: gwapiv1a2.GroupName, + Kind: gatewayapi.KindGateway, + Namespace: gwapiv1a2.Namespace("test-ns"), + }, + }, + To: []gwapiv1a2.ReferenceGrantTo{ + { + Group: corev1.GroupName, + Kind: gatewayapi.KindSecret, + }, + }, + }, + }, + }, + }, + { + name: "gateway with one https listener and one different namespace secret with specific referencegrant", + gw: &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: "test-gc", + Listeners: []gwapiv1b1.Listener{ + { + Name: "tls", + Port: gwapiv1b1.PortNumber(int32(443)), + Protocol: gwapiv1b1.HTTPSProtocolType, + TLS: &gwapiv1b1.GatewayTLSConfig{ + Mode: gatewayapi.TLSModeTypePtr(gwapiv1b1.TLSModeTerminate), + CertificateRefs: []gwapiv1b1.SecretObjectReference{ + { + Name: gwapiv1b1.ObjectName("test-secret"), + Namespace: gatewayapi.NamespacePtr("test-ns2"), + }, + }, + }, + }, + }, + }, + }, + secrets: []corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-ns2", + ResourceVersion: "1", + }, + }, + }, + refGrants: []gwapiv1a2.ReferenceGrant{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "ReferenceGrant", + APIVersion: gwapiv1a2.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-refgrant", + Namespace: "test-ns2", + }, + Spec: gwapiv1a2.ReferenceGrantSpec{ + From: []gwapiv1a2.ReferenceGrantFrom{ + { + Group: gwapiv1a2.GroupName, + Kind: gatewayapi.KindGateway, + Namespace: gwapiv1a2.Namespace("test-ns"), + }, + }, + To: []gwapiv1a2.ReferenceGrantTo{ + { + Group: corev1.GroupName, + Kind: gatewayapi.KindSecret, + Name: gatewayapi.ObjectNamePtr("test-secret"), + }, + }, + }, + }, + }, + }, + } + + // Create the reconciler. + r := new(gatewayReconciler) + ctx := context.Background() + + for i := range testCases { + tc := testCases[i] + t.Run(tc.name, func(t *testing.T) { + var objs []client.Object + for j := range tc.secrets { + objs = append(objs, &tc.secrets[j]) + } + for k := range tc.refGrants { + objs = append(objs, &tc.refGrants[k]) + } + r.client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(objs...).Build() + secrets, refGrants, err := r.secretsAndRefGrantsForGateway(ctx, tc.gw) + require.NoError(t, err) + require.Equal(t, tc.secrets, secrets) + require.Equal(t, tc.refGrants, refGrants) + }) + } +} diff --git a/internal/provider/kubernetes/testdata/in/referencegrant-experimental-crd.yaml b/internal/provider/kubernetes/testdata/in/referencegrant-experimental-crd.yaml new file mode 100644 index 00000000000..0cc826ee422 --- /dev/null +++ b/internal/provider/kubernetes/testdata/in/referencegrant-experimental-crd.yaml @@ -0,0 +1,149 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/1086 + gateway.networking.k8s.io/bundle-version: v0.5.0 + gateway.networking.k8s.io/channel: experimental + creationTimestamp: null + name: referencegrants.gateway.networking.k8s.io +spec: + group: gateway.networking.k8s.io + names: + categories: + - gateway-api + kind: ReferenceGrant + listKind: ReferenceGrantList + plural: referencegrants + shortNames: + - refgrant + singular: referencegrant + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: "ReferenceGrant identifies kinds of resources in other namespaces + that are trusted to reference the specified kinds of resources in the same + namespace as the policy. \n Each ReferenceGrant can be used to represent + a unique trust relationship. Additional Reference Grants can be used to + add to the set of trusted sources of inbound references for the namespace + they are defined within. \n All cross-namespace references in Gateway API + (with the exception of cross-namespace Gateway-route attachment) require + a ReferenceGrant. \n Support: Core" + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of ReferenceGrant. + properties: + from: + description: "From describes the trusted namespaces and kinds that + can reference the resources described in \"To\". Each entry in this + list must be considered to be an additional place that references + can be valid from, or to put this another way, entries must be combined + using OR. \n Support: Core" + items: + description: ReferenceGrantFrom describes trusted namespaces and + kinds. + properties: + group: + description: "Group is the group of the referent. When empty, + the Kubernetes core API group is inferred. \n Support: Core" + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: "Kind is the kind of the referent. Although implementations + may support additional resources, the following types are + part of the \"Core\" support level for this field. \n When + used to permit a SecretObjectReference: \n * Gateway \n When + used to permit a BackendObjectReference: \n * HTTPRoute * + TCPRoute * TLSRoute * UDPRoute" + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + namespace: + description: "Namespace is the namespace of the referent. \n + Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - namespace + type: object + maxItems: 16 + minItems: 1 + type: array + to: + description: "To describes the resources that may be referenced by + the resources described in \"From\". Each entry in this list must + be considered to be an additional place that references can be valid + to, or to put this another way, entries must be combined using OR. + \n Support: Core" + items: + description: ReferenceGrantTo describes what Kinds are allowed as + targets of the references. + properties: + group: + description: "Group is the group of the referent. When empty, + the Kubernetes core API group is inferred. \n Support: Core" + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: "Kind is the kind of the referent. Although implementations + may support additional resources, the following types are + part of the \"Core\" support level for this field: \n * Secret + when used to permit a SecretObjectReference * Service when + used to permit a BackendObjectReference" + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. When unspecified, + this policy refers to all resources of the specified Group + and Kind in the local namespace. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + type: object + maxItems: 16 + minItems: 1 + type: array + required: + - from + - to + type: object + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index e69f5586b71..3f82da9d040 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -36,6 +36,7 @@ func TestGatewayAPIConformance(t *testing.T) { v1alpha2.PortNumber(int32(83)), v1alpha2.PortNumber(int32(84)), }, + SupportedFeatures: []suite.SupportedFeature{suite.SupportReferenceGrant}, }) cSuite.Setup(t) egTests := []suite.ConformanceTest{ @@ -50,6 +51,12 @@ func TestGatewayAPIConformance(t *testing.T) { tests.HTTPRouteInvalidNonExistentBackendRef, tests.HTTPRouteInvalidBackendRefUnknownKind, tests.HTTPRouteInvalidCrossNamespaceBackendRef, + tests.HTTPRouteHeaderMatching, + tests.GatewaySecretReferenceGrantAllInNamespace, + tests.GatewaySecretReferenceGrantSpecific, + // Uncomment when https://github.com/envoyproxy/gateway/issues/538 is fixed. + /*tests.GatewaySecretMissingReferenceGrant, + tests.GatewaySecretInvalidReferenceGrant,*/ } cSuite.Run(t, egTests) From 5801e0bd5d1226e0b37f85e1e636fbf37bf95691 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Wed, 12 Oct 2022 04:39:27 +0800 Subject: [PATCH 011/113] chore: run conformance tests with identical ports (#534) * chore: run conformance tests with identical ports Signed-off-by: bitliu Signed-off-by: bitliu Signed-off-by: Arko Dasgupta Co-authored-by: Arko Dasgupta --- .github/workflows/build_and_test.yaml | 2 +- test/conformance/conformance_test.go | 32 ++++++++++++++++++--------- tools/make/kube.mk | 4 +++- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index b4586c3cada..ba6a038dee0 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -49,7 +49,7 @@ jobs: # conformance - name: Run Conformance Tests - run: make conformance + run: CONFORMANCE_UNIQUE_PORTS=false make conformance # build and push image - name: Login to DockerHub diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index 3f82da9d040..b0aa079ea3c 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -4,6 +4,7 @@ package conformance import ( + "flag" "testing" "github.com/stretchr/testify/require" @@ -15,7 +16,11 @@ import ( "sigs.k8s.io/gateway-api/conformance/utils/suite" ) +var useUniquePorts = flag.Bool("use-unique-ports", true, "whether to use unique ports") + func TestGatewayAPIConformance(t *testing.T) { + flag.Parse() + cfg, err := config.GetConfig() require.NoError(t, err) @@ -24,18 +29,23 @@ func TestGatewayAPIConformance(t *testing.T) { require.NoError(t, v1alpha2.AddToScheme(client.Scheme())) + validUniqueListenerPorts := []v1alpha2.PortNumber{ + v1alpha2.PortNumber(int32(80)), + v1alpha2.PortNumber(int32(81)), + v1alpha2.PortNumber(int32(82)), + v1alpha2.PortNumber(int32(83)), + } + + if !*useUniquePorts { + validUniqueListenerPorts = []v1alpha2.PortNumber{} + } + cSuite := suite.New(suite.Options{ - Client: client, - GatewayClassName: *flags.GatewayClassName, - Debug: *flags.ShowDebug, - CleanupBaseResources: *flags.CleanupBaseResources, - ValidUniqueListenerPorts: []v1alpha2.PortNumber{ - v1alpha2.PortNumber(int32(80)), - v1alpha2.PortNumber(int32(81)), - v1alpha2.PortNumber(int32(82)), - v1alpha2.PortNumber(int32(83)), - v1alpha2.PortNumber(int32(84)), - }, + Client: client, + GatewayClassName: *flags.GatewayClassName, + Debug: *flags.ShowDebug, + CleanupBaseResources: *flags.CleanupBaseResources, + ValidUniqueListenerPorts: validUniqueListenerPorts, SupportedFeatures: []suite.SupportedFeature{suite.SupportReferenceGrant}, }) cSuite.Setup(t) diff --git a/tools/make/kube.mk b/tools/make/kube.mk index f458e7f7d55..820ebd68ab7 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -4,6 +4,8 @@ ENVTEST_K8S_VERSION ?= 1.24.1 # For more details, see https://gateway-api.sigs.k8s.io/guides/getting-started/#installing-gateway-api GATEWAY_API_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/gateway-api) +CONFORMANCE_UNIQUE_PORTS ?= true + # Set Kubernetes Resources Directory Path ifeq ($(origin KUBE_PROVIDER_DIR),undefined) KUBE_PROVIDER_DIR := $(ROOT_DIR)/internal/provider/kubernetes/config @@ -97,7 +99,7 @@ run-conformance: ## Run Gateway API conformance. kubectl wait --timeout=5m -n gateway-system deployment/gateway-api-admission-server --for=condition=Available kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f internal/provider/kubernetes/config/samples/gatewayclass.yaml - go test -v -tags conformance ./test/conformance --gateway-class=envoy-gateway --debug=true + go test -v -tags conformance ./test/conformance --gateway-class=envoy-gateway --debug=true --use-unique-ports=$(CONFORMANCE_UNIQUE_PORTS) .PHONY: delete-cluster delete-cluster: $(tools/kind) ## Delete kind cluster. From 6ad3c0c749569483db19be1fa9e8d2feb8070dcb Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Tue, 11 Oct 2022 13:39:41 -0700 Subject: [PATCH 012/113] rm duplicate HTTPRouteHeaderMatching test (#541) Signed-off-by: Arko Dasgupta --- test/conformance/conformance_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index b0aa079ea3c..9490e96bf5a 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -61,7 +61,6 @@ func TestGatewayAPIConformance(t *testing.T) { tests.HTTPRouteInvalidNonExistentBackendRef, tests.HTTPRouteInvalidBackendRefUnknownKind, tests.HTTPRouteInvalidCrossNamespaceBackendRef, - tests.HTTPRouteHeaderMatching, tests.GatewaySecretReferenceGrantAllInNamespace, tests.GatewaySecretReferenceGrantSpecific, // Uncomment when https://github.com/envoyproxy/gateway/issues/538 is fixed. From 03be9f4d3ff795a44a841f6ae332833c63f9a0d0 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Tue, 11 Oct 2022 13:53:39 -0700 Subject: [PATCH 013/113] Consolidates Quickstart Manifests (#511) Signed-off-by: danehans Signed-off-by: danehans --- .github/workflows/release.yaml | 1 + examples/kubernetes/gateway.yaml | 10 ---- examples/kubernetes/gatewayclass.yaml | 6 -- examples/kubernetes/httpbin.yaml | 43 -------------- examples/kubernetes/httproute.yaml | 20 ------- examples/kubernetes/quickstart.yaml | 82 +++++++++++++++++++++++++++ tools/hack/release-manifests.sh | 3 + tools/make/kube.mk | 10 +--- 8 files changed, 88 insertions(+), 87 deletions(-) delete mode 100644 examples/kubernetes/gateway.yaml delete mode 100644 examples/kubernetes/gatewayclass.yaml delete mode 100644 examples/kubernetes/httpbin.yaml delete mode 100644 examples/kubernetes/httproute.yaml create mode 100644 examples/kubernetes/quickstart.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6760efc924b..87010d63ec5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,5 +35,6 @@ jobs: with: files: | release-artifacts/install.yaml + release-artifacts/quickstart.yaml diff --git a/examples/kubernetes/gateway.yaml b/examples/kubernetes/gateway.yaml deleted file mode 100644 index bceb8e21027..00000000000 --- a/examples/kubernetes/gateway.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: Gateway -metadata: - name: eg -spec: - gatewayClassName: eg - listeners: - - name: http - protocol: HTTP - port: 8080 diff --git a/examples/kubernetes/gatewayclass.yaml b/examples/kubernetes/gatewayclass.yaml deleted file mode 100644 index 9a2fbab002a..00000000000 --- a/examples/kubernetes/gatewayclass.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: GatewayClass -metadata: - name: eg -spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/examples/kubernetes/httpbin.yaml b/examples/kubernetes/httpbin.yaml deleted file mode 100644 index 53da0f062bb..00000000000 --- a/examples/kubernetes/httpbin.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: httpbin ---- -apiVersion: v1 -kind: Service -metadata: - name: httpbin - labels: - app: httpbin - service: httpbin -spec: - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app: httpbin ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: httpbin -spec: - replicas: 1 - selector: - matchLabels: - app: httpbin - version: v1 - template: - metadata: - labels: - app: httpbin - version: v1 - spec: - serviceAccountName: httpbin - containers: - - image: docker.io/kennethreitz/httpbin - imagePullPolicy: IfNotPresent - name: httpbin - ports: - - containerPort: 80 diff --git a/examples/kubernetes/httproute.yaml b/examples/kubernetes/httproute.yaml deleted file mode 100644 index bc85544d6c4..00000000000 --- a/examples/kubernetes/httproute.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: HTTPRoute -metadata: - name: httpbin -spec: - parentRefs: - - name: eg - hostnames: - - "www.example.com" - rules: - - backendRefs: - - group: "" - kind: Service - name: httpbin - port: 80 - weight: 1 - matches: - - path: - type: PathPrefix - value: / diff --git a/examples/kubernetes/quickstart.yaml b/examples/kubernetes/quickstart.yaml new file mode 100644 index 00000000000..e542d205497 --- /dev/null +++ b/examples/kubernetes/quickstart.yaml @@ -0,0 +1,82 @@ +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: Gateway +metadata: + name: eg +spec: + gatewayClassName: eg + listeners: + - name: http + protocol: HTTP + port: 8080 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: httpbin +--- +apiVersion: v1 +kind: Service +metadata: + name: httpbin + labels: + app: httpbin + service: httpbin +spec: + ports: + - name: http + port: 80 + targetPort: 80 + selector: + app: httpbin +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + version: v1 + template: + metadata: + labels: + app: httpbin + version: v1 + spec: + serviceAccountName: httpbin + containers: + - image: docker.io/kennethreitz/httpbin + imagePullPolicy: IfNotPresent + name: httpbin + ports: + - containerPort: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: HTTPRoute +metadata: + name: httpbin +spec: + parentRefs: + - name: eg + hostnames: + - "www.example.com" + rules: + - backendRefs: + - group: "" + kind: Service + name: httpbin + port: 80 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/tools/hack/release-manifests.sh b/tools/hack/release-manifests.sh index 0f501676785..6928ea7ca25 100755 --- a/tools/hack/release-manifests.sh +++ b/tools/hack/release-manifests.sh @@ -33,6 +33,9 @@ ${KUSTOMIZE} build > release-artifacts/install.yaml echo "Generated:" release-artifacts/install.yaml +# Copy the quickstart manifest +cp examples/kubernetes/quickstart.yaml release-artifacts/quickstart.yaml + # Update the image in the Envoy Gateway deployment manifest. [[ -n "${TAG}" ]] && run::sed \ "-es|image: envoyproxy/gateway-dev:.*$|image: envoyproxy/gateway:${TAG}|" \ diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 820ebd68ab7..2785ad5793f 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -63,10 +63,7 @@ kube-undeploy: kube-uninstall ## Uninstall the Envoy Gateway controller into the .PHONY: kube-demo kube-demo: ## Deploy a demo backend service, gatewayclass, gateway and httproute resource and test the configuration. - kubectl apply -f examples/kubernetes/httpbin.yaml - kubectl apply -f examples/kubernetes/gatewayclass.yaml - kubectl apply -f examples/kubernetes/gateway.yaml - kubectl apply -f examples/kubernetes/httproute.yaml + kubectl apply -f examples/kubernetes/quickstart.yaml @echo "\nPort forward to the Envoy service using the command below" @echo "kubectl -n envoy-gateway-system port-forward service/envoy-default-eg 8888:8080 &" @echo "\nCurl the app through Envoy proxy using the command below" @@ -74,10 +71,7 @@ kube-demo: ## Deploy a demo backend service, gatewayclass, gateway and httproute .PHONY: kube-demo-undeploy kube-demo-undeploy: ## Uninstall the Kubernetes resources installed from the `make kube-demo` command. - kubectl delete -f examples/kubernetes/httproute.yaml - kubectl delete -f examples/kubernetes/gateway.yaml - kubectl delete -f examples/kubernetes/gatewayclass.yaml - kubectl delete -f examples/kubernetes/httpbin.yaml + kubectl delete -f examples/kubernetes/quickstart.yaml .PHONY: run-kube-local run-kube-local: build kube-install ## Run Envoy Gateway locally. From 5d5c28eb0911968b82f355376767e97fd89dcdf7 Mon Sep 17 00:00:00 2001 From: Alice Wasko Date: Tue, 11 Oct 2022 15:15:32 -0700 Subject: [PATCH 014/113] Update example manifest apiversion (#544) update example manifest apiversion Signed-off-by: AliceProxy --- examples/kubernetes/quickstart.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/kubernetes/quickstart.yaml b/examples/kubernetes/quickstart.yaml index e542d205497..b045025c85a 100644 --- a/examples/kubernetes/quickstart.yaml +++ b/examples/kubernetes/quickstart.yaml @@ -1,11 +1,11 @@ -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1beta1 kind: GatewayClass metadata: name: eg spec: controllerName: gateway.envoyproxy.io/gatewayclass-controller --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1beta1 kind: Gateway metadata: name: eg @@ -60,7 +60,7 @@ spec: ports: - containerPort: 80 --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1beta1 kind: HTTPRoute metadata: name: httpbin From 2fb3ca8954705dabbcfa08007b658655fcb60f46 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Tue, 11 Oct 2022 17:57:11 -0700 Subject: [PATCH 015/113] Updates ParentRef Status Conditions (#495) * Updates ParentRef Status Conditions Signed-off-by: danehans * Resolved @skriss 10-11-22 Feedback Signed-off-by: danehans Signed-off-by: danehans --- ...er-duplicate-add-multiple-filters.out.yaml | 4 -- ...with-header-filter-duplicate-adds.out.yaml | 5 --- ...duplicate-remove-multiple-filters.out.yaml | 4 -- ...h-header-filter-duplicate-removes.out.yaml | 4 -- ...-with-header-filter-empty-headers.out.yaml | 4 -- ...ith-header-filter-invalid-headers.out.yaml | 4 -- ...th-header-filter-no-valid-headers.out.yaml | 4 -- ...direct-filter-invalid-filter-type.out.yaml | 4 -- ...th-redirect-filter-invalid-scheme.out.yaml | 4 -- ...th-redirect-filter-invalid-status.out.yaml | 4 -- internal/gatewayapi/translator.go | 42 ++++++------------- 11 files changed, 12 insertions(+), 71 deletions(-) diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml index 1408cc0eae4..10a378a589b 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml @@ -73,10 +73,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "RequestHeaderModifier Filter already configures request header: add-header-1 to be added, ignoring second entry" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml index 011724c4560..c23b5425678 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml @@ -83,11 +83,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - # Currently only one invalid value status will be set. If there are multiple, then only the latest is displayed until that issue is resolved. - message: "RequestHeaderModifier Filter already configures request header: set-header-4 to be added/set, ignoring second entry" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml index 003288773a2..01c33fa8d5e 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml @@ -69,10 +69,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "RequestHeaderModifier Filter already configures request header: rem-header-1 to be removed, ignoring second entry" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml index 7123200aff5..bde4b77b6f7 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml @@ -64,10 +64,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "RequestHeaderModifier Filter already configures request header: some-header-1 to be removed, ignoring second entry" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml index 238545928dd..ae0cd811cad 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml @@ -69,10 +69,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "RequestHeaderModifier Filter cannot set a header with an empty name" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml index e8d0ff09547..29a05459042 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml @@ -69,10 +69,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "RequestHeaderModifier Filter cannot set headers with a '/' or ':' character in them. Header: 'example:1'" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml index a63a937bb2b..3892e0bc6c9 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml @@ -64,10 +64,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "RequestHeaderModifier Filter did not provide valid configuration to add/set/remove any headers" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml index 7a210ae376e..3127021f040 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml @@ -63,10 +63,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "Unknown custom filter type: UnsupportedType" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml index a663120def3..630f9fb75ec 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml @@ -63,10 +63,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "Scheme: unknown is unsupported, only 'https' and 'http' are supported" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml index 998d188aa3a..dbb2df4af0c 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml @@ -63,10 +63,6 @@ httpRoutes: status: "True" reason: Accepted message: Route is accepted - - type: ResolvedRefs - status: "False" - reason: UnsupportedValue - message: "Status code 666 is invalid, only 302 and 301 are supported" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 71e3ef9a115..0d4db0be7e0 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -779,7 +779,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Can't have two redirects for the same route if redirectResponse != nil { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, "Cannot configure multiple requestRedirect filters for a single HTTPRouteRule", @@ -801,7 +801,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } else { errMsg := fmt.Sprintf("Scheme: %s is unsupported, only 'https' and 'http' are supported", *redirect.Scheme) parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, errMsg, @@ -813,7 +813,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways if redirect.Hostname != nil { if err := isValidHostname(string(*redirect.Hostname)); err != nil { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, err.Error(), @@ -842,7 +842,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways default: errMsg := fmt.Sprintf("Redirect path type: %s is invalid, only \"ReplaceFullPath\" and \"ReplacePrefixMatch\" are supported", redirect.Path.Type) parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, errMsg, @@ -859,7 +859,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } else { errMsg := fmt.Sprintf("Status code %d is invalid, only 302 and 301 are supported", redirectCode) parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, errMsg, @@ -891,7 +891,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways emptyFilterConfig = false if addHeader.Name == "" { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, "RequestHeaderModifier Filter cannot add a header with an empty name", @@ -902,7 +902,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Per Gateway API specification on HTTPHeaderName, : and / are invalid characters in header names if strings.Contains(string(addHeader.Name), "/") || strings.Contains(string(addHeader.Name), ":") { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, fmt.Sprintf("RequestHeaderModifier Filter cannot set headers with a '/' or ':' character in them. Header: %q", string(addHeader.Name)), @@ -920,12 +920,6 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } if !canAddHeader { - parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, - metav1.ConditionFalse, - v1beta1.RouteReasonUnsupportedValue, - fmt.Sprintf("RequestHeaderModifier Filter already configures request header: %s to be added, ignoring second entry", headerKey), - ) continue } @@ -948,7 +942,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways if setHeader.Name == "" { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, "RequestHeaderModifier Filter cannot set a header with an empty name", @@ -958,7 +952,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Per Gateway API specification on HTTPHeaderName, : and / are invalid characters in header names if strings.Contains(string(setHeader.Name), "/") || strings.Contains(string(setHeader.Name), ":") { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, fmt.Sprintf("RequestHeaderModifier Filter cannot set headers with a '/' or ':' character in them. Header: '%s'", string(setHeader.Name)), @@ -976,12 +970,6 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } } if !canAddHeader { - parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, - metav1.ConditionFalse, - v1beta1.RouteReasonUnsupportedValue, - fmt.Sprintf("RequestHeaderModifier Filter already configures request header: %s to be added/set, ignoring second entry", headerKey), - ) continue } newHeader := ir.AddHeader{ @@ -1004,7 +992,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways for _, removedHeader := range headersToRemove { if removedHeader == "" { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, "RequestHeaderModifier Filter cannot remove a header with an empty name", @@ -1020,12 +1008,6 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } } if !canRemHeader { - parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, - metav1.ConditionFalse, - v1beta1.RouteReasonUnsupportedValue, - fmt.Sprintf("RequestHeaderModifier Filter already configures request header: %s to be removed, ignoring second entry", removedHeader), - ) continue } @@ -1037,7 +1019,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Update the status if the filter failed to configure any valid headers to add/remove if len(addRequestHeaders) == 0 && len(removeRequestHeaders) == 0 && !emptyFilterConfig { parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, "RequestHeaderModifier Filter did not provide valid configuration to add/set/remove any headers", @@ -1048,7 +1030,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways // Instead, requests that would have been processed by that filter MUST receive a HTTP error response." errMsg := fmt.Sprintf("Unknown custom filter type: %s", filter.Type) parentRef.SetCondition(httpRoute, - v1beta1.RouteConditionResolvedRefs, + v1beta1.RouteConditionAccepted, metav1.ConditionFalse, v1beta1.RouteReasonUnsupportedValue, errMsg, From 01a638c07dd4dab0a5a1df85b8c5cf2582f5bae2 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 12 Oct 2022 17:35:06 -0700 Subject: [PATCH 016/113] Removes run-kube-local Target (#552) Signed-off-by: danehans --- tools/make/kube.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 2785ad5793f..1601659ac4e 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -73,9 +73,10 @@ kube-demo: ## Deploy a demo backend service, gatewayclass, gateway and httproute kube-demo-undeploy: ## Uninstall the Kubernetes resources installed from the `make kube-demo` command. kubectl delete -f examples/kubernetes/quickstart.yaml -.PHONY: run-kube-local -run-kube-local: build kube-install ## Run Envoy Gateway locally. - tools/hack/run-kube-local.sh +# Uncomment when https://github.com/envoyproxy/gateway/issues/256 is fixed. +#.PHONY: run-kube-local +#run-kube-local: build kube-install ## Run Envoy Gateway locally. +# tools/hack/run-kube-local.sh .PHONY: conformance conformance: create-cluster kube-install-image kube-deploy run-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. From 0be1aef9248c54e16836d0f3a960df147bc09f11 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Fri, 14 Oct 2022 01:59:01 +0800 Subject: [PATCH 017/113] chore: add local shellcheck tools (#521) Signed-off-by: bitliu --- tools/make/golang.mk | 2 +- tools/make/lint.mk | 7 ++++--- tools/make/tools.mk | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tools/make/golang.mk b/tools/make/golang.mk index e4aaf1cce8b..d6fc6a97d05 100644 --- a/tools/make/golang.mk +++ b/tools/make/golang.mk @@ -12,7 +12,7 @@ GO_VERSION = $(shell grep -oE "^go [[:digit:]]*\.[[:digit:]]*" go.mod | cut -d' # Build the target binary in target platform. # The pattern of build.% is `build.{Platform}.{Command}`. # If we want to build envoy-gateway in linux amd64 platform, -# just execute make build.linux_amd64.envoy-gateway. +# just execute make go.build.linux_amd64.envoy-gateway. .PHONY: go.build.% go.build.%: $(eval COMMAND := $(word 2,$(subst ., ,$*))) diff --git a/tools/make/lint.mk b/tools/make/lint.mk index ba00b6a1296..9aca38fe94e 100644 --- a/tools/make/lint.mk +++ b/tools/make/lint.mk @@ -58,12 +58,13 @@ lint.whitenoise: $(tools/whitenoise) @echo Running WhiteNoise linter ... $(tools/whitenoise) -# GitHub has shellcheck pre-installed + .PHONY: lint.shellcheck lint: lint.shellcheck -lint.shellcheck: +lint-deps: $(tools/shellcheck) +lint.shellcheck: $(tools/shellcheck) @echo Running Shellcheck linter ... - @shellcheck tools/hack/*.sh + $(tools/shellcheck) tools/hack/*.sh .PHONY: gen-check gen-check: generate manifests diff --git a/tools/make/tools.mk b/tools/make/tools.mk index 2d52dceeaf4..6495120ea43 100644 --- a/tools/make/tools.mk +++ b/tools/make/tools.mk @@ -33,3 +33,23 @@ $(tools.bindir)/%.d/venv: $(tools.srcdir)/%/requirements.txt $@/bin/pip3 install -r $< || (rm -rf $@; exit 1) $(tools.bindir)/%: $(tools.bindir)/%.d/venv ln -sf $*.d/venv/bin/$* $@ + +ifneq ($(GOOS),windows) +# Shellcheck +# ========== +# +tools/shellcheck = $(tools.bindir)/shellcheck +SHELLCHECK_VERSION=0.8.0 +SHELLCHECK_ARCH=$(shell uname -m) +# shellcheck uses the same binary on Intel and Apple Silicon Mac. +ifeq ($(GOOS),darwin) +SHELLCHECK_ARCH=x86_64 +endif +SHELLCHECK_TXZ = https://github.com/koalaman/shellcheck/releases/download/v$(SHELLCHECK_VERSION)/shellcheck-v$(SHELLCHECK_VERSION).$(GOOS).$(SHELLCHECK_ARCH).tar.xz +tools/bin/$(notdir $(SHELLCHECK_TXZ)): + mkdir -p $(@D) + curl -sfL $(SHELLCHECK_TXZ) -o $@ +%/bin/shellcheck: %/bin/$(notdir $(SHELLCHECK_TXZ)) + mkdir -p $(@D) + tar -C $(@D) -Jxmf $< --strip-components=1 shellcheck-v$(SHELLCHECK_VERSION)/shellcheck +endif From 2103195e28257a564d3aea674f9cb401cb037032 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 13 Oct 2022 12:24:23 -0700 Subject: [PATCH 018/113] Fixes URL in Quickstart Doc (#560) Signed-off-by: danehans --- docs/user/QUICKSTART.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/QUICKSTART.md b/docs/user/QUICKSTART.md index 62142310a60..980d1e371cd 100644 --- a/docs/user/QUICKSTART.md +++ b/docs/user/QUICKSTART.md @@ -29,7 +29,7 @@ kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2 Create the Gateway: ```shell -kubectl apply -f https://raw.githubusercontent.com//envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/gateway.yaml +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/gateway.yaml ``` Create the HTTPRoute to route traffic through Envoy proxy to the example app: From 75f93c141539fc100ccb644a2e459cf42a0064fa Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Fri, 14 Oct 2022 09:34:05 +0800 Subject: [PATCH 019/113] chore: set IMAGE to release-manifests (#553) Signed-off-by: bitliu --- .github/workflows/build_and_test.yaml | 4 +- .github/workflows/release.yaml | 4 +- kustomization.yaml | 4 -- tools/hack/release-manifests.sh | 44 ---------------------- tools/make/kube.mk | 53 ++++++++++++++------------- 5 files changed, 31 insertions(+), 78 deletions(-) delete mode 100644 kustomization.yaml delete mode 100755 tools/hack/release-manifests.sh diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index ba6a038dee0..b43e355c566 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -64,9 +64,9 @@ jobs: - name: Build and Push EG Commit Image if: github.event_name == 'push' # tag is set to the short SHA of the commit - run: make image.push.multiarch PLATFORMS="linux_amd64 linux_arm64" + run: make image.push.multiarch PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev - name: Build and Push EG Latest Image if: github.event_name == 'push' && github.ref == 'refs/heads/main' # tag is set to `latest` when pushing to main branch - run: make image.push.multiarch TAG=latest PLATFORMS="linux_amd64 linux_arm64" + run: make image.push.multiarch TAG=latest PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 87010d63ec5..69646ecc6e1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,7 +28,7 @@ jobs: skopeo copy --all docker://docker.io/envoyproxy/gateway-dev:${{ steps.vars.outputs.sha_short }} docker://docker.io/envoyproxy/gateway:${{ steps.vars.outputs.release_tag }} - name: Generate Release Manifests - run: make release-manifests TAG=${{ steps.vars.outputs.release_tag}} + run: make generate-manifests IMAGE=envoyproxy/gateway TAG=${{ steps.vars.outputs.release_tag}} OUTPUT_DIR=release-artifacts - name: Upload Release Manifests uses: softprops/action-gh-release@v1 @@ -36,5 +36,3 @@ jobs: files: | release-artifacts/install.yaml release-artifacts/quickstart.yaml - - diff --git a/kustomization.yaml b/kustomization.yaml deleted file mode 100644 index 60f835a20fa..00000000000 --- a/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: - - release-artifacts/gatewayapi-crds.yaml - - release-artifacts/envoy-gateway.yaml - - release-artifacts/infra-manager-rbac.yaml diff --git a/tools/hack/release-manifests.sh b/tools/hack/release-manifests.sh deleted file mode 100755 index 6928ea7ca25..00000000000 --- a/tools/hack/release-manifests.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -readonly KUSTOMIZE=${KUSTOMIZE:-tools/bin/kustomize} -readonly GATEWAY_API_VERSION="$1" -readonly TAG="$2" - -mkdir -p release-artifacts/ - -# Wrap sed to deal with GNU and BSD sed flags. -run::sed() { - if sed --version &1 | grep -q GNU; then - # GNU sed - sed -i "$@" - else - # assume BSD sed - sed -i '' "$@" - fi -} - -# Download the supported Gateway API CRDs that will be supported by the release. -curl -sLo release-artifacts/gatewayapi-crds.yaml https://github.com/kubernetes-sigs/gateway-api/releases/download/"${GATEWAY_API_VERSION}"/experimental-install.yaml - -echo "Added:" release-artifacts/gatewayapi-crds.yaml - -# Generate the envoy gateway installation manifest supported by the release. -${KUSTOMIZE} build internal/provider/kubernetes/config/default > release-artifacts/envoy-gateway.yaml -${KUSTOMIZE} build internal/infrastructure/kubernetes/config/rbac > release-artifacts/infra-manager-rbac.yaml -${KUSTOMIZE} build > release-artifacts/install.yaml - -echo "Generated:" release-artifacts/install.yaml - -# Copy the quickstart manifest -cp examples/kubernetes/quickstart.yaml release-artifacts/quickstart.yaml - -# Update the image in the Envoy Gateway deployment manifest. -[[ -n "${TAG}" ]] && run::sed \ - "-es|image: envoyproxy/gateway-dev:.*$|image: envoyproxy/gateway:${TAG}|" \ - "release-artifacts/install.yaml" - -echo "Updated the envoy gateway image:" release-artifacts/install.yaml diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 1601659ac4e..e0c8d3dd4b0 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -4,6 +4,8 @@ ENVTEST_K8S_VERSION ?= 1.24.1 # For more details, see https://gateway-api.sigs.k8s.io/guides/getting-started/#installing-gateway-api GATEWAY_API_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/gateway-api) +GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml + CONFORMANCE_UNIQUE_PORTS ?= true # Set Kubernetes Resources Directory Path @@ -36,30 +38,13 @@ ifndef ignore-not-found ignore-not-found = true endif -.PHONY: kube-install -kube-install: manifests $(tools/kustomize) ## Install Envoy Gateway CRDs into the Kubernetes cluster specified in ~/.kube/config. - mkdir -pv $(OUTPUT_DIR)/manifests/provider - cp -r $(KUBE_PROVIDER_DIR) $(OUTPUT_DIR)/manifests/provider - mkdir -pv $(OUTPUT_DIR)/manifests/infra - cp -r $(KUBE_INFRA_DIR) $(OUTPUT_DIR)/manifests/infra - $(tools/kustomize) build $(OUTPUT_DIR)/manifests/provider/config/crd | kubectl apply -f - - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml - -.PHONY: kube-uninstall -kube-uninstall: manifests $(tools/kustomize) ## Uninstall Envoy Gateway CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml --ignore-not-found=$(ignore-not-found) - .PHONY: kube-deploy -kube-deploy: kube-install ## Install Envoy Gateway controller into the Kubernetes cluster specified in ~/.kube/config. - cd $(OUTPUT_DIR)/manifests/provider/config/envoy-gateway && $(ROOT_DIR)/$(tools/kustomize) edit set image envoyproxy/gateway-dev=$(IMAGE):$(TAG) - $(tools/kustomize) build $(OUTPUT_DIR)/manifests/provider/config/default | kubectl apply -f - - $(tools/kustomize) build $(OUTPUT_DIR)/manifests/infra/config/rbac | kubectl apply -f - +kube-deploy: manifests $(tools/kustomize) generate-manifests ## Install Envoy Gateway into the Kubernetes cluster specified in ~/.kube/config. + kubectl apply -f $(OUTPUT_DIR)/install.yaml .PHONY: kube-undeploy -kube-undeploy: kube-uninstall ## Uninstall the Envoy Gateway controller into the Kubernetes cluster specified in ~/.kube/config. - $(tools/kustomize) build $(OUTPUT_DIR)/manifests/provider/config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - rm -rf $(OUTPUT_DIR)/manifests/provider - rm -rf $(OUTPUT_DIR)/manifests/infra +kube-undeploy: manifests $(tools/kustomize) ## Uninstall the Envoy Gateway into the Kubernetes cluster specified in ~/.kube/config. + kubectl delete --ignore-not-found=$(ignore-not-found) -f $(OUTPUT_DIR)/install.yaml .PHONY: kube-demo kube-demo: ## Deploy a demo backend service, gatewayclass, gateway and httproute resource and test the configuration. @@ -71,7 +56,7 @@ kube-demo: ## Deploy a demo backend service, gatewayclass, gateway and httproute .PHONY: kube-demo-undeploy kube-demo-undeploy: ## Uninstall the Kubernetes resources installed from the `make kube-demo` command. - kubectl delete -f examples/kubernetes/quickstart.yaml + kubectl delete -f examples/kubernetes/quickstart.yaml --ignore-not-found=$(ignore-not-found) # Uncomment when https://github.com/envoyproxy/gateway/issues/256 is fixed. #.PHONY: run-kube-local @@ -100,6 +85,24 @@ run-conformance: ## Run Gateway API conformance. delete-cluster: $(tools/kind) ## Delete kind cluster. $(tools/kind) delete cluster --name envoy-gateway -.PHONY: release-manifests -release-manifests: $(tools/kustomize) ## Generate Kubernetes release manifests. - tools/hack/release-manifests.sh $(GATEWAY_API_VERSION) $(TAG) +.PHONY: generate-manifests +generate-manifests: $(tools/kustomize) ## Generate Kubernetes release manifests. + @echo "\033[36m===========> Generating kubernetes manifests\033[0m" + mkdir -p $(OUTPUT_DIR)/ + curl -sLo $(OUTPUT_DIR)/gatewayapi-crds.yaml ${GATEWAY_RELEASE_URL} + @echo "\033[36m===========> Added: $(OUTPUT_DIR)/gatewayapi-crds.yaml\033[0m" + mkdir -pv $(OUTPUT_DIR)/manifests/provider + cp -r $(KUBE_PROVIDER_DIR) $(OUTPUT_DIR)/manifests/provider + mkdir -pv $(OUTPUT_DIR)/manifests/infra + cp -r $(KUBE_INFRA_DIR) $(OUTPUT_DIR)/manifests/infra + cd $(OUTPUT_DIR)/manifests/provider/config/envoy-gateway && $(ROOT_DIR)/$(tools/kustomize) edit set image envoyproxy/gateway-dev=$(IMAGE):$(TAG) + $(tools/kustomize) build $(OUTPUT_DIR)/manifests/provider/config/default > $(OUTPUT_DIR)/envoy-gateway.yaml + $(tools/kustomize) build $(OUTPUT_DIR)/manifests/infra/config/rbac > $(OUTPUT_DIR)/infra-manager-rbac.yaml + touch $(OUTPUT_DIR)/kustomization.yaml + cd $(OUTPUT_DIR) && $(ROOT_DIR)/$(tools/kustomize) edit add resource ./envoy-gateway.yaml + cd $(OUTPUT_DIR) && $(ROOT_DIR)/$(tools/kustomize) edit add resource ./infra-manager-rbac.yaml + cd $(OUTPUT_DIR) && $(ROOT_DIR)/$(tools/kustomize) edit add resource ./gatewayapi-crds.yaml + $(tools/kustomize) build $(OUTPUT_DIR) > $(OUTPUT_DIR)/install.yaml + @echo "\033[36m===========> Added: $(OUTPUT_DIR)/install.yaml\033[0m" + cp examples/kubernetes/quickstart.yaml $(OUTPUT_DIR)/quickstart.yaml + @echo "\033[36m===========> Added: $(OUTPUT_DIR)/quickstart.yaml\033[0m" From 61f0999942c303c344ef3623eaba7c4332b78237 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Fri, 14 Oct 2022 09:54:01 +0800 Subject: [PATCH 020/113] feat: set mgr to interface (#530) Signed-off-by: bitliu Signed-off-by: bitliu --- internal/infrastructure/kubernetes/infra.go | 4 ++-- .../infrastructure/kubernetes/infra_test.go | 4 ++-- internal/infrastructure/manager.go | 21 ++++++++++++------- internal/infrastructure/runner/runner.go | 4 ++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index 402c7119ec6..39e6eb4074b 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -28,8 +28,8 @@ func NewInfra(cli client.Client) *Infra { } } -// CreateInfra creates the managed kube infra, if it doesn't exist. -func (i *Infra) CreateInfra(ctx context.Context, infra *ir.Infra) error { +// CreateOrUpdateInfra creates the managed kube infra, if it doesn't exist. +func (i *Infra) CreateOrUpdateInfra(ctx context.Context, infra *ir.Infra) error { if infra == nil { return errors.New("infra ir is nil") } diff --git a/internal/infrastructure/kubernetes/infra_test.go b/internal/infrastructure/kubernetes/infra_test.go index a8a697a475f..a13374468bb 100644 --- a/internal/infrastructure/kubernetes/infra_test.go +++ b/internal/infrastructure/kubernetes/infra_test.go @@ -60,8 +60,8 @@ func TestCreateInfra(t *testing.T) { Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), Namespace: "default", } - // Create the proxy infra. - err := kube.CreateInfra(context.Background(), tc.in) + // Create or update the proxy infra. + err := kube.CreateOrUpdateInfra(context.Background(), tc.in) if !tc.expect { require.Error(t, err) } else { diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index 4b86068fffc..8ed44db7a34 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -1,6 +1,7 @@ package infrastructure import ( + "context" "fmt" "sigs.k8s.io/controller-runtime/pkg/client" @@ -10,26 +11,30 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes" + "github.com/envoyproxy/gateway/internal/ir" ) +var _ Manager = (*kubernetes.Infra)(nil) + // Manager provides the scaffolding for managing infrastructure. -type Manager struct { - // TODO: create a common infra interface - *kubernetes.Infra +type Manager interface { + // CreateOrUpdateInfra creates or updates infra. + CreateOrUpdateInfra(ctx context.Context, infra *ir.Infra) error + // DeleteInfra deletes infra + DeleteInfra(ctx context.Context, infra *ir.Infra) error } // NewManager returns a new infrastructure Manager. -func NewManager(cfg *config.Server) (*Manager, error) { - mgr := new(Manager) - +func NewManager(cfg *config.Server) (Manager, error) { + var mgr Manager if cfg.EnvoyGateway.Provider.Type == v1alpha1.ProviderTypeKubernetes { cli, err := client.New(clicfg.GetConfigOrDie(), client.Options{Scheme: envoygateway.GetScheme()}) if err != nil { return nil, err } - mgr.Infra = kubernetes.NewInfra(cli) + mgr = kubernetes.NewInfra(cli) } else { - // Kube is the only supported provider type. + // Kube is the only supported provider type for now. return nil, fmt.Errorf("unsupported provider type %v", cfg.EnvoyGateway.Provider.Type) } diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index 3101212701e..d51c2fd1cbf 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -15,7 +15,7 @@ type Config struct { type Runner struct { Config - mgr *infrastructure.Manager + mgr infrastructure.Manager } func (r *Runner) Name() string { @@ -52,7 +52,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { } } else { // Manage the proxy infra. - if err := r.mgr.CreateInfra(ctx, val); err != nil { + if err := r.mgr.CreateOrUpdateInfra(ctx, val); err != nil { r.Logger.Error(err, "failed to create new infra") } } From 6a1ced2fdd4ce86a8910f9837778e8d413f89322 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Sat, 15 Oct 2022 01:42:45 +0800 Subject: [PATCH 021/113] fix: remove deprecated set-output in release pipeline (#568) fix: remove deprecated set-output Signed-off-by: bitliu --- .github/workflows/release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 69646ecc6e1..7b1e345892b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,8 +15,8 @@ jobs: id: vars shell: bash run: | - echo "::set-output name=release_tag::$(echo ${GITHUB_REF##*/})" - echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + echo "release_tag=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - name: Login to DockerHub uses: docker/login-action@v2 @@ -25,10 +25,10 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Retag and push existing gateway-dev image run: | - skopeo copy --all docker://docker.io/envoyproxy/gateway-dev:${{ steps.vars.outputs.sha_short }} docker://docker.io/envoyproxy/gateway:${{ steps.vars.outputs.release_tag }} + skopeo copy --all docker://docker.io/envoyproxy/gateway-dev:${{ env.sha_short }} docker://docker.io/envoyproxy/gateway:${{ env.release_tag }} - name: Generate Release Manifests - run: make generate-manifests IMAGE=envoyproxy/gateway TAG=${{ steps.vars.outputs.release_tag}} OUTPUT_DIR=release-artifacts + run: make generate-manifests IMAGE=envoyproxy/gateway TAG=${{ env.release_tag}} OUTPUT_DIR=release-artifacts - name: Upload Release Manifests uses: softprops/action-gh-release@v1 From d36d140675826107b0e847efff3d77aab5c8e10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Geijer=20Haeggstr=C3=B6m?= Date: Fri, 14 Oct 2022 19:43:02 +0200 Subject: [PATCH 022/113] docs: correcting route ref, system design (#570) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Geijer Haeggström --- docs/design/SYSTEM_DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/SYSTEM_DESIGN.md b/docs/design/SYSTEM_DESIGN.md index 46bdf547236..2b741bc110c 100644 --- a/docs/design/SYSTEM_DESIGN.md +++ b/docs/design/SYSTEM_DESIGN.md @@ -136,7 +136,7 @@ The draft for this document is [here][draft_design]. [crf]: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#filters-optional [gwapi_conflicts]: https://gateway-api.sigs.k8s.io/concepts/guidelines/#conflicts [listener]: https://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/listeners#config-listeners -[route]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route.proto#config-route-v3-routeconfiguration +[route]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-route [be_ref]: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#backendrefs-optional [cluster]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster [draft_design]: https://docs.google.com/document/d/1riyTPPYuvNzIhBdrAX8dpfxTmcobWZDSYTTB5NeybuY/edit From 319bcb93cc074497d6f1a25d4f7fd80c77016c39 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 14 Oct 2022 12:07:49 -0600 Subject: [PATCH 023/113] Fix intermittent failures (#557) * .gitignore: Ignore `vendor/` directories Signed-off-by: Luke Shumaker * xds translator: Fix racy startup If the watchable.Map has content in it already when .Subscribe() is called on it, then those initial entries won't have a snapshot.Updates entry in that first snapshot. For the first snapshot we just need to iterate over snapshot.State. Signed-off-by: Luke Shumaker * provider tests: Fix running the test multiple times controller-runtime.SetupSignalHandler() panics if called more than once in a process. So running the test multiple times (`go test -count=2`) reliably causes the test to panic. So don't use ctrl.SetupSignalHandler() in unit tests. Signed-off-by: Luke Shumaker * Add and use a new watchutil.HandleSubscription function As the added godoc comment says, "This is better than iterating over snapshot.Updates because it handles the case where the the watchable.Map already contains entries before .Subscribe is called." The generalizes the fix that I made in the XDS translator. Signed-off-by: Luke Shumaker * docs: Add a bit to watching.md about HandleSubscription Signed-off-by: Luke Shumaker * Move HandleSubscription et al. around per Arko's feedback I was going to do a type alias for `watchable.Update`, but: internal/message/watchutil.go:7:6: generic type cannot be alias So I just defined a new child type, which is fine because there aren't any methods on Update. Signed-off-by: Luke Shumaker --- .gitignore | 3 + docs/design/watching.md | 17 ++++-- internal/infrastructure/runner/runner.go | 10 ++-- internal/message/watchutil.go | 34 +++++++++++ internal/message/watchutil_test.go | 56 +++++++++++++++++++ internal/provider/kubernetes/gateway.go | 12 ++-- internal/provider/kubernetes/httproute.go | 12 ++-- internal/provider/kubernetes/tlsroute.go | 12 ++-- internal/provider/runner/runner_test.go | 5 +- internal/xds/server/runner/runner.go | 11 ++-- internal/xds/translator/runner/runner.go | 12 ++-- internal/xds/translator/runner/runner_test.go | 2 - 12 files changed, 138 insertions(+), 48 deletions(-) create mode 100644 internal/message/watchutil.go create mode 100644 internal/message/watchutil_test.go diff --git a/.gitignore b/.gitignore index d8a1fb4f35e..3dcae053694 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ release-artifacts/ # Outputs coverage.xml + +# `go mod vendor` +vendor/ diff --git a/docs/design/watching.md b/docs/design/watching.md index 19cb785edfd..e505b6280a0 100644 --- a/docs/design/watching.md +++ b/docs/design/watching.md @@ -90,12 +90,17 @@ handy way to know when to run, `.Load` and friends can be used without subscribi There can be any number of subscribers. For that matter, there can be any number of publishers `.Store`ing things, but it's probably wise to just have one publisher for each map. -The channel returned from `.Subscribe` is immediately readable with a snapshot of the map as it existed when -`.Subscribe` was called; after that initial read it becomes readable again whenever `.Store` or `.Delete` mutates the -map. If multiple mutations happen between reads, they are coalesced in to one snapshot to be read; the `snapshot.State` -is the most-recent full state, and `snapshot.Updates` is a listing of each of the mutations that cause this snapshot to -be different than the last-read one. This way subscribers don't need to worry about a backlog accumulating if they -can't keep up with the rate of changes from the publisher. +The channel returned from `.Subscribe` **is immediately readable** with a snapshot of the map as it existed when +`.Subscribe` was called; and becomes readable again whenever `.Store` or `.Delete` mutates the map. If multiple +mutations happen between reads (or if mutations happen between `.Subscribe` and the first read), they are coalesced in +to one snapshot to be read; the `snapshot.State` is the most-recent full state, and `snapshot.Updates` is a listing of +each of the mutations that cause this snapshot to be different than the last-read one. This way subscribers don't need +to worry about a backlog accumulating if they can't keep up with the rate of changes from the publisher. + +If the map contains anything before `.Subscribe` is called, that very first read won't include `snapshot.Updates` +entries for those pre-existing items; if you are working with `snapshot.Update` instead of `snapshot.State`, then you +must add special handling for your first read. We have a utility function `./internal/message.HandleSubscription` to +help with this. ### other notes diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index d51c2fd1cbf..ce5413b7d80 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -5,6 +5,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/infrastructure" + "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/message" ) @@ -41,9 +42,8 @@ func (r *Runner) Start(ctx context.Context) error { func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Subscribe to resources - for snapshot := range r.InfraIR.Subscribe(ctx) { - r.Logger.Info("received a notification") - for _, update := range snapshot.Updates { + message.HandleSubscription(r.InfraIR.Subscribe(ctx), + func(update message.Update[string, *ir.Infra]) { val := update.Value if update.Delete { @@ -56,7 +56,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { r.Logger.Error(err, "failed to create new infra") } } - } - } + }, + ) r.Logger.Info("subscriber shutting down") } diff --git a/internal/message/watchutil.go b/internal/message/watchutil.go new file mode 100644 index 00000000000..40f22cb23f2 --- /dev/null +++ b/internal/message/watchutil.go @@ -0,0 +1,34 @@ +package message + +import ( + "github.com/telepresenceio/watchable" +) + +type Update[K comparable, V any] watchable.Update[K, V] + +// HandleSubscription takes a channel returned by +// watchable.Map.Subscribe() (or .SubscribeSubset()), and calls the +// given function for each initial value in the map, and for any +// updates. +// +// This is better than simply iterating over snapshot.Updates because +// it handles the case where the the watchable.Map already contains +// entries before .Subscribe is called. +func HandleSubscription[K comparable, V any]( + subscription <-chan watchable.Snapshot[K, V], + handle func(Update[K, V]), +) { + if snapshot, ok := <-subscription; ok { + for k, v := range snapshot.State { + handle(Update[K, V]{ + Key: k, + Value: v, + }) + } + } + for snapshot := range subscription { + for _, update := range snapshot.Updates { + handle(Update[K, V](update)) + } + } +} diff --git a/internal/message/watchutil_test.go b/internal/message/watchutil_test.go new file mode 100644 index 00000000000..abc0ccb5d2d --- /dev/null +++ b/internal/message/watchutil_test.go @@ -0,0 +1,56 @@ +package message_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/telepresenceio/watchable" + + "github.com/envoyproxy/gateway/internal/message" +) + +func TestHandleSubscriptionAlreadyClosed(t *testing.T) { + ch := make(chan watchable.Snapshot[string, any]) + close(ch) + + var calls int + message.HandleSubscription[string, any]( + ch, + func(message.Update[string, any]) { calls++ }, + ) + assert.Equal(t, 0, calls) +} + +func TestHandleSubscriptionAlreadyInitialized(t *testing.T) { + var m watchable.Map[string, any] + m.Store("foo", "bar") + + endCtx, end := context.WithCancel(context.Background()) + go func() { + <-endCtx.Done() + m.Store("baz", "qux") + m.Delete("qux") // no-op + m.Store("foo", "bar") // no-op + m.Delete("baz") + time.Sleep(100 * time.Millisecond) + m.Close() + }() + + var storeCalls int + var deleteCalls int + message.HandleSubscription[string, any]( + m.Subscribe(context.Background()), + func(update message.Update[string, any]) { + end() + if update.Delete { + deleteCalls++ + } else { + storeCalls++ + } + }, + ) + assert.Equal(t, 2, storeCalls) + assert.Equal(t, 1, deleteCalls) +} diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index 4bd2ba007dc..89482d70dbc 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -622,13 +622,11 @@ func (r *gatewayReconciler) envoyDeploymentForGateway(ctx context.Context, gatew // Kubernetes API Server func (r *gatewayReconciler) subscribeAndUpdateStatus(ctx context.Context) { // Subscribe to resources - for snapshot := range r.resources.GatewayStatuses.Subscribe(ctx) { - r.log.Info("received a status notification") - updates := snapshot.Updates - for _, update := range updates { + message.HandleSubscription(r.resources.GatewayStatuses.Subscribe(ctx), + func(update message.Update[types.NamespacedName, *gwapiv1b1.Gateway]) { // skip delete updates. if update.Delete { - continue + return } key := update.Key val := update.Value @@ -645,8 +643,8 @@ func (r *gatewayReconciler) subscribeAndUpdateStatus(ctx context.Context) { return gCopy }), }) - } - } + }, + ) r.log.Info("status subscriber shutting down") } diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go index 253e787b318..e01a77a1924 100644 --- a/internal/provider/kubernetes/httproute.go +++ b/internal/provider/kubernetes/httproute.go @@ -318,13 +318,11 @@ func validateBackendRef(ref *gwapiv1b1.HTTPBackendRef) error { // Kubernetes API Server func (r *httpRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { // Subscribe to resources - for snapshot := range r.resources.HTTPRouteStatuses.Subscribe(ctx) { - r.log.Info("received a status notification") - updates := snapshot.Updates - for _, update := range updates { + message.HandleSubscription(r.resources.HTTPRouteStatuses.Subscribe(ctx), + func(update message.Update[types.NamespacedName, *gwapiv1b1.HTTPRoute]) { // skip delete updates. if update.Delete { - continue + return } key := update.Key val := update.Value @@ -341,7 +339,7 @@ func (r *httpRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { return hCopy }), }) - } - } + }, + ) r.log.Info("status subscriber shutting down") } diff --git a/internal/provider/kubernetes/tlsroute.go b/internal/provider/kubernetes/tlsroute.go index ff4296b53a5..f78a661b1d8 100644 --- a/internal/provider/kubernetes/tlsroute.go +++ b/internal/provider/kubernetes/tlsroute.go @@ -297,13 +297,11 @@ func validateTLSRouteBackendRef(ref *gwapiv1a2.BackendRef) error { // Kubernetes API Server func (r *tlsRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { // Subscribe to resources - for snapshot := range r.resources.TLSRouteStatuses.Subscribe(ctx) { - r.log.Info("received a status notification") - updates := snapshot.Updates - for _, update := range updates { + message.HandleSubscription(r.resources.TLSRouteStatuses.Subscribe(ctx), + func(update message.Update[types.NamespacedName, *gwapiv1a2.TLSRoute]) { // skip delete updates. if update.Delete { - continue + return } key := update.Key val := update.Value @@ -320,7 +318,7 @@ func (r *tlsRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { return tCopy }), }) - } - } + }, + ) r.log.Info("status subscriber shutting down") } diff --git a/internal/provider/runner/runner_test.go b/internal/provider/runner/runner_test.go index 8411e7664ed..44bf29c2a68 100644 --- a/internal/provider/runner/runner_test.go +++ b/internal/provider/runner/runner_test.go @@ -1,11 +1,11 @@ package runner import ( + "context" "testing" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" @@ -51,7 +51,8 @@ func TestStart(t *testing.T) { ProviderResources: new(message.ProviderResources), }, } - ctx := ctrl.SetupSignalHandler() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) err := runner.Start(ctx) if tc.expect { require.NoError(t, err) diff --git a/internal/xds/server/runner/runner.go b/internal/xds/server/runner/runner.go index 5c3a9988976..38f359b6535 100644 --- a/internal/xds/server/runner/runner.go +++ b/internal/xds/server/runner/runner.go @@ -16,6 +16,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" "github.com/envoyproxy/gateway/internal/xds/cache" + xdstypes "github.com/envoyproxy/gateway/internal/xds/types" controlplane_service_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/service/cluster/v3" controlplane_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" controlplane_service_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/service/endpoint/v3" @@ -112,10 +113,8 @@ func registerServer(srv controlplane_server_v3.Server, g *grpc.Server) { func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Subscribe to resources - for snapshot := range r.Xds.Subscribe(ctx) { - r.Logger.Info("received a notification") - // Load all resources required for translation - for _, update := range snapshot.Updates { + message.HandleSubscription(r.Xds.Subscribe(ctx), + func(update message.Update[string, *xdstypes.ResourceVersionTable]) { key := update.Key val := update.Value @@ -129,8 +128,8 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { if err != nil { r.Logger.Error(err, "failed to generate a snapshot") } - } - } + }, + ) r.Logger.Info("subscriber shutting down") diff --git a/internal/xds/translator/runner/runner.go b/internal/xds/translator/runner/runner.go index daa84570876..515436b708a 100644 --- a/internal/xds/translator/runner/runner.go +++ b/internal/xds/translator/runner/runner.go @@ -4,6 +4,7 @@ import ( "context" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/message" "github.com/envoyproxy/gateway/internal/xds/translator" ) @@ -36,10 +37,9 @@ func (r *Runner) Start(ctx context.Context) error { func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Subscribe to resources - for snapshot := range r.XdsIR.Subscribe(ctx) { - r.Logger.Info("received a notification") - updates := snapshot.Updates - for _, update := range updates { + message.HandleSubscription(r.XdsIR.Subscribe(ctx), + func(update message.Update[string, *ir.Xds]) { + r.Logger.Info("received an update") key := update.Key val := update.Value @@ -55,7 +55,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { r.Xds.Store(key, result) } } - } - } + }, + ) r.Logger.Info("subscriber shutting down") } diff --git a/internal/xds/translator/runner/runner_test.go b/internal/xds/translator/runner/runner_test.go index 36de62fab59..b6649b1cf74 100644 --- a/internal/xds/translator/runner/runner_test.go +++ b/internal/xds/translator/runner/runner_test.go @@ -14,8 +14,6 @@ import ( ) func TestRunner(t *testing.T) { - // Remove once https://github.com/envoyproxy/gateway/issues/504 is completed. - t.Skip() // Setup xdsIR := new(message.XdsIR) xds := new(message.Xds) From 39456ca2bd9647547a824529e62520a37da1276d Mon Sep 17 00:00:00 2001 From: Alice Wasko Date: Fri, 14 Oct 2022 14:44:18 -0700 Subject: [PATCH 024/113] Hash ir/infra resources (#559) * infra: hash resources with long names Signed-off-by: AliceProxy * add tests for hashing resources Signed-off-by: AliceProxy * hashing: replace sha1 with sha256 Signed-off-by: AliceProxy * hashing: only use 8 chars Signed-off-by: AliceProxy * ir/infra: always hash resource names Signed-off-by: AliceProxy * update all test manifests with hashed names Signed-off-by: AliceProxy * only hash necessary resources Signed-off-by: AliceProxy * update test manifests Signed-off-by: AliceProxy Signed-off-by: AliceProxy --- internal/envoygateway/config/config.go | 8 +- .../infrastructure/kubernetes/configmap.go | 4 +- .../kubernetes/configmap_test.go | 7 +- .../infrastructure/kubernetes/deployment.go | 4 +- internal/infrastructure/kubernetes/service.go | 4 +- .../kubernetes/serviceaccount.go | 9 +- .../kubernetes/serviceaccount_test.go | 49 ++++++++- internal/provider/kubernetes/gateway.go | 6 +- internal/provider/kubernetes/gateway_test.go | 17 +++ .../provider/kubernetes/kubernetes_test.go | 100 ++++++++++++++++++ internal/provider/utils/utils.go | 17 +++ 11 files changed, 204 insertions(+), 21 deletions(-) diff --git a/internal/envoygateway/config/config.go b/internal/envoygateway/config/config.go index bddabf33b7e..c8f8562becd 100644 --- a/internal/envoygateway/config/config.go +++ b/internal/envoygateway/config/config.go @@ -12,12 +12,8 @@ const ( EnvoyGatewayNamespace = "envoy-gateway-system" // EnvoyGatewayServiceName is the name of the Envoy Gateway service. EnvoyGatewayServiceName = "envoy-gateway" - // EnvoyConfigMapPrefix is the prefix applied to the Envoy ConfigMap. - EnvoyConfigMapPrefix = "envoy" - // EnvoyServicePrefix is the prefix applied to the Envoy Service. - EnvoyServicePrefix = "envoy" - // EnvoyDeploymentPrefix is the prefix applied to the Envoy Deployment. - EnvoyDeploymentPrefix = "envoy" + // EnvoyPrefix is the prefix applied to the Envoy ConfigMap, Service, Deployment, and ServiceAccount. + EnvoyPrefix = "envoy" ) // Server wraps the EnvoyGateway configuration and additional parameters diff --git a/internal/infrastructure/kubernetes/configmap.go b/internal/infrastructure/kubernetes/configmap.go index 0fe6e08553d..ce24122cc50 100644 --- a/internal/infrastructure/kubernetes/configmap.go +++ b/internal/infrastructure/kubernetes/configmap.go @@ -13,6 +13,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/provider/utils" ) const ( @@ -113,5 +114,6 @@ func (i *Infra) deleteConfigMap(ctx context.Context, infra *ir.Infra) error { } func expectedConfigMapName(proxyName string) string { - return fmt.Sprintf("%s-%s", config.EnvoyConfigMapPrefix, proxyName) + cMapName := utils.GetHashedName(proxyName) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, cMapName) } diff --git a/internal/infrastructure/kubernetes/configmap_test.go b/internal/infrastructure/kubernetes/configmap_test.go index 5051e81bbd4..a240412cfdd 100644 --- a/internal/infrastructure/kubernetes/configmap_test.go +++ b/internal/infrastructure/kubernetes/configmap_test.go @@ -22,6 +22,7 @@ func TestExpectedConfigMap(t *testing.T) { cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() kube := NewInfra(cli) infra := ir.NewInfra() + infra.Proxy.Name = "test" // An infra without Gateway owner labels should trigger @@ -35,7 +36,7 @@ func TestExpectedConfigMap(t *testing.T) { cm, err := kube.expectedConfigMap(infra) require.NoError(t, err) - require.Equal(t, "envoy-test", cm.Name) + require.Equal(t, "envoy-test-74657374", cm.Name) require.Equal(t, "envoy-gateway-system", cm.Namespace) require.Contains(t, cm.Data, sdsCAFilename) assert.Equal(t, sdsCAConfigMapData, cm.Data[sdsCAFilename]) @@ -65,7 +66,7 @@ func TestCreateOrUpdateConfigMap(t *testing.T) { expect: &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: config.EnvoyGatewayNamespace, - Name: "envoy-test", + Name: "envoy-test-74657374", Labels: map[string]string{ "app.gateway.envoyproxy.io/name": "envoy", gatewayapi.OwningGatewayNamespaceLabel: "default", @@ -92,7 +93,7 @@ func TestCreateOrUpdateConfigMap(t *testing.T) { expect: &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: config.EnvoyGatewayNamespace, - Name: "envoy-test", + Name: "envoy-test-74657374", Labels: map[string]string{ "app.gateway.envoyproxy.io/name": "envoy", gatewayapi.OwningGatewayNamespaceLabel: "default", diff --git a/internal/infrastructure/kubernetes/deployment.go b/internal/infrastructure/kubernetes/deployment.go index ef6796a87fc..d3d19d3253c 100644 --- a/internal/infrastructure/kubernetes/deployment.go +++ b/internal/infrastructure/kubernetes/deployment.go @@ -18,6 +18,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/provider/utils" xdsrunner "github.com/envoyproxy/gateway/internal/xds/server/runner" ) @@ -94,7 +95,8 @@ func (b *bootstrapConfig) render() error { } func expectedDeploymentName(proxyName string) string { - return fmt.Sprintf("%s-%s", config.EnvoyDeploymentPrefix, proxyName) + deploymentName := utils.GetHashedName(proxyName) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, deploymentName) } // expectedDeployment returns the expected Deployment based on the provided infra. diff --git a/internal/infrastructure/kubernetes/service.go b/internal/infrastructure/kubernetes/service.go index 4a0f9e12b1b..fd93f81c08b 100644 --- a/internal/infrastructure/kubernetes/service.go +++ b/internal/infrastructure/kubernetes/service.go @@ -14,10 +14,12 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/provider/utils" ) func expectedServiceName(proxyName string) string { - return fmt.Sprintf("%s-%s", config.EnvoyServicePrefix, proxyName) + svcName := utils.GetHashedName(proxyName) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, svcName) } // expectedService returns the expected Service based on the provided infra. diff --git a/internal/infrastructure/kubernetes/serviceaccount.go b/internal/infrastructure/kubernetes/serviceaccount.go index 2a20ee742d8..3a40f61822d 100644 --- a/internal/infrastructure/kubernetes/serviceaccount.go +++ b/internal/infrastructure/kubernetes/serviceaccount.go @@ -9,16 +9,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" -) - -const ( - envoyServiceAccountPrefix = "envoy" + "github.com/envoyproxy/gateway/internal/provider/utils" ) func expectedServiceAccountName(proxyName string) string { - return fmt.Sprintf("%s-%s", envoyServiceAccountPrefix, proxyName) + svcActName := utils.GetHashedName(proxyName) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, svcActName) } // expectedServiceAccount returns the expected proxy serviceAccount. diff --git a/internal/infrastructure/kubernetes/serviceaccount_test.go b/internal/infrastructure/kubernetes/serviceaccount_test.go index 0eaddbcb3cb..d6299d0f128 100644 --- a/internal/infrastructure/kubernetes/serviceaccount_test.go +++ b/internal/infrastructure/kubernetes/serviceaccount_test.go @@ -73,7 +73,7 @@ func TestCreateOrUpdateServiceAccount(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: "test", - Name: "envoy-test", + Name: "envoy-test-74657374", Labels: map[string]string{ "app.gateway.envoyproxy.io/name": "envoy", gatewayapi.OwningGatewayNamespaceLabel: "default", @@ -118,7 +118,52 @@ func TestCreateOrUpdateServiceAccount(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: "test", - Name: "envoy-test", + Name: "envoy-test-74657374", + Labels: map[string]string{ + "app.gateway.envoyproxy.io/name": "envoy", + gatewayapi.OwningGatewayNamespaceLabel: "default", + gatewayapi.OwningGatewayNameLabel: "gateway-1", + }, + }, + }, + }, + { + name: "hashed-name", + ns: "test", + in: &ir.Infra{ + Proxy: &ir.ProxyInfra{ + Name: "very-long-name-that-will-be-hashed-and-cut-off-because-its-too-long", + Metadata: &ir.InfraMetadata{ + Labels: map[string]string{ + gatewayapi.OwningGatewayNamespaceLabel: "default", + gatewayapi.OwningGatewayNameLabel: "gateway-1", + }, + }, + }, + }, + current: &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "very-long-name-that-will-be-hashed-and-cut-off-because-its-too-long", + Labels: map[string]string{ + "app.gateway.envoyproxy.io/name": "envoy", + gatewayapi.OwningGatewayNamespaceLabel: "default", + gatewayapi.OwningGatewayNameLabel: "gateway-1", + }, + }, + }, + want: &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "envoy-very-long-name-that-will-be-hashed-and-cut-off-b-76657279", Labels: map[string]string{ "app.gateway.envoyproxy.io/name": "envoy", gatewayapi.OwningGatewayNamespaceLabel: "default", diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index 89482d70dbc..e740202bdfb 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -649,9 +649,11 @@ func (r *gatewayReconciler) subscribeAndUpdateStatus(ctx context.Context) { } func infraServiceName(gateway *gwapiv1b1.Gateway) string { - return fmt.Sprintf("%s-%s-%s", config.EnvoyServicePrefix, gateway.Namespace, gateway.Name) + infraName := utils.GetHashedName(fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name)) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, infraName) } func infraDeploymentName(gateway *gwapiv1b1.Gateway) string { - return fmt.Sprintf("%s-%s-%s", config.EnvoyDeploymentPrefix, gateway.Namespace, gateway.Name) + infraName := utils.GetHashedName(fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name)) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, infraName) } diff --git a/internal/provider/kubernetes/gateway_test.go b/internal/provider/kubernetes/gateway_test.go index 2240f3692b6..8f4248e8cfa 100644 --- a/internal/provider/kubernetes/gateway_test.go +++ b/internal/provider/kubernetes/gateway_test.go @@ -99,6 +99,23 @@ func TestGatewayHasMatchingController(t *testing.T) { }, expect: false, }, + { + name: "matching but very long name", + obj: &gwapiv1b1.Gateway{ + TypeMeta: metav1.TypeMeta{ + Kind: "Gateway", + APIVersion: fmt.Sprintf("%s/%s", gwapiv1b1.GroupName, gwapiv1b1.GroupVersion.Version), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "superdupermegalongnamethatisridiculouslylongandwaylongerthanitshouldeverbeinsideofkubernetes", + Namespace: "test", + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: gwapiv1b1.ObjectName(match.Name), + }, + }, + expect: true, + }, } // Create the reconciler. diff --git a/internal/provider/kubernetes/kubernetes_test.go b/internal/provider/kubernetes/kubernetes_test.go index 655a0eee7c7..2d6cacbc334 100644 --- a/internal/provider/kubernetes/kubernetes_test.go +++ b/internal/provider/kubernetes/kubernetes_test.go @@ -254,6 +254,106 @@ func testGatewayScheduledStatus(ctx context.Context, t *testing.T, provider *Pro assert.Equal(t, gw.Spec, gws.Spec) } +// Test that even when resources such as the Service/Deployment get hashed names (because of a gateway with a very long name) +func testLongNameHashedResources(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { + cli := provider.manager.GetClient() + + gc := getGatewayClass("envoy-gateway-class") + require.NoError(t, cli.Create(ctx, gc)) + + // Ensure the GatewayClass reports "Ready". + require.Eventually(t, func() bool { + if err := cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc); err != nil { + return false + } + + for _, cond := range gc.Status.Conditions { + if cond.Type == string(gwapiv1b1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { + return true + } + } + + return false + }, defaultWait, defaultTick) + + defer func() { + require.NoError(t, cli.Delete(ctx, gc)) + }() + + // Create the namespace for the Gateway under test. + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway"}} + require.NoError(t, cli.Create(ctx, ns)) + + gw := &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gatewaywithaverylongnamethatwillresultinhashedresources", + Namespace: ns.Name, + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: gwapiv1b1.ObjectName(gc.Name), + Listeners: []gwapiv1b1.Listener{ + { + Name: "test", + Port: gwapiv1b1.PortNumber(int32(8080)), + Protocol: gwapiv1b1.HTTPProtocolType, + }, + }, + }, + } + require.NoError(t, cli.Create(ctx, gw)) + + // Ensure the Gateway is ready and gets an address. + ready := false + hasAddress := false + require.Eventually(t, func() bool { + if err := cli.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: gw.Name}, gw); err != nil { + return false + } + + for _, cond := range gw.Status.Conditions { + fmt.Printf("Condition: %v", cond) + if cond.Type == string(gwapiv1b1.GatewayConditionReady) && cond.Status == metav1.ConditionTrue { + ready = true + } + } + + if gw.Status.Addresses != nil { + hasAddress = len(gw.Status.Addresses) >= 1 + } + + return ready && hasAddress + }, defaultWait, defaultTick) + + defer func() { + require.NoError(t, cli.Delete(ctx, gw)) + }() + + // Ensure the gatewayclass has been finalized. + require.NoError(t, cli.Get(ctx, types.NamespacedName{Name: gc.Name}, gc)) + require.Contains(t, gc.Finalizers, gatewayClassFinalizer) + + // Ensure the number of Gateways in the Gateway resource table is as expected. + require.Eventually(t, func() bool { + return resources.Gateways.Len() == 1 + }, defaultWait, defaultTick) + + // Ensure the test Gateway in the Gateway resources is as expected. + key := types.NamespacedName{ + Namespace: gw.Namespace, + Name: gw.Name, + } + require.Eventually(t, func() bool { + return cli.Get(ctx, key, gw) == nil + }, defaultWait, defaultTick) + gws, _ := resources.Gateways.Load(key) + // Only check if the spec is equal + // The watchable map will not store a resource + // with an updated status if the spec has not changed + // to eliminate this endless loop: + // reconcile->store->translate->update-status->reconcile + assert.Equal(t, gw.Spec, gws.Spec) +} + func testHTTPRoute(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { cli := provider.manager.GetClient() diff --git a/internal/provider/utils/utils.go b/internal/provider/utils/utils.go index 715e5588bce..f736356a5ae 100644 --- a/internal/provider/utils/utils.go +++ b/internal/provider/utils/utils.go @@ -1,6 +1,10 @@ package utils import ( + "crypto/sha256" + "fmt" + "strings" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -12,3 +16,16 @@ func NamespacedName(obj client.Object) types.NamespacedName { Name: obj.GetName(), } } + +// Returns a partially hashed name for the string including up to 48 characters of the original name before the hash +func GetHashedName(name string) string { + + h := sha256.New() // Using sha256 instead of sha1 due to Blocklisted import crypto/sha1: weak cryptographic primitive (gosec) + hsha := h.Sum([]byte(name)) + hashedName := strings.ToLower(fmt.Sprintf("%x", hsha)) + + if len(name) > 48 { + return fmt.Sprintf("%s-%s", name[0:48], hashedName[0:8]) + } + return fmt.Sprintf("%s-%s", name, hashedName[0:8]) +} From 6a387d37b18288b509ec9253bbde6bb987e16cdf Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 09:07:21 -0700 Subject: [PATCH 025/113] Adds Release Doc (#325) * Adds Release Doc Signed-off-by: danehans * Resolves Arko and Luke review feedback Signed-off-by: danehans * Removes the step to link release notes Signed-off-by: danehans Signed-off-by: danehans --- docs/dev/RELEASE.md | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/dev/RELEASE.md diff --git a/docs/dev/RELEASE.md b/docs/dev/RELEASE.md new file mode 100644 index 00000000000..674e5c35e6c --- /dev/null +++ b/docs/dev/RELEASE.md @@ -0,0 +1,89 @@ +## Introduction +This document guides maintainers through the process of creating an Envoy Gateway release. + +## Prerequisites +- Permissions to push to the Envoy Gateway repository. + +## Creating a Minor Release + +1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. +2. Create the release notes corresponding to the release number. Reference previous [release notes][] + for additional details. +3. Submit a [Pull Request][] to merge the release notes into the main branch. This should be the last commit to main + before cutting the release. +4. Create a new release branch from `main`. The release branch should be named + `release/v${MAJOR_VERSION}.${MINOR_VERSION}.0`, e.g. `release/v0.3.0`. + ```shell + git checkout -b release/v0.3.0 + ``` +5. Push the branch to the Envoy Gateway repo. +6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. See [PR 481][] as + a reference for the required changes. +7. Sign, commit, and push your changes to your fork. Send a PR to get your changes merged into the release branch. + Do not proceed until your PR is merged. +8. Confirm that the [release workflow][] for your PR completed successfully. +9. Tag the head of your release branch with the release tag. For example: + ```shell + git tag -a v0.3.0 -m 'Envoy Gateway v0.3.0 Release' + ``` +10. Push the tag to the Envoy Gateway repository. + ```shell + git push --tags + ``` +11. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +12. Confirm that the [release workflow][] completed successfully. +13. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +14. Confirm that the [release][] was created. +15. Confirm that the steps in the [Quickstart Guide][] work as expected. +16. [Generate][] the GitHub changelog. +18. Submit a PR to merge the Quickstart Guide changes from the release branch into the main branch. +19. If you find any bugs in this process, please create an issue. + +## Creating a Release Candidate + +1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. +2. Create the release notes corresponding to the release candidate that summarizes the changes included in the + release candidate. Reference previous [release notes][] for additional details. +3. Submit a [Pull Request][] to merge the changelog into the main branch. This should be the last commit to main + before cutting the release candidate. +4. Tag the head of the main branch with the release candidate number. The tag should be named + `v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER}`. For example: + ```shell + git tag -a v0.3.0-rc.1 -m 'Envoy Gateway v0.3.0-rc.1 Release Candidate' + ``` +5. Push the tag to the Envoy Gateway repository. + ```shell + git push --tags + ``` +6. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +7. Confirm that the [release workflow][] completed successfully. +8. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +9. Confirm that the [release][] was created. +10. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test + the quickstart steps using the release candidate by manually updating the links. +11. [Generate][] the GitHub changelog. +13. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. +14. If you find any bugs in this process, please create an issue. + +## Announcing the Release +It's important that the world knows about the release. Follow the steps to announce the release. +1. Set the release information in the Envoy Gateway Slack channel. For example: + ```shell + Envoy Gateway v0.3.0 has been released: https://github.com/envoyproxy/gateway/releases/tag/v0.3.0 + ``` +2. Send a message to the Envoy Gateway Slack channel. For example: + ```shell + I am pleased to announce the release of Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0. The release would not be + possible without all the support from the Envoy Gateway community... + ``` + Include a sentence or two that highlights key aspects of the release. + +[release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes +[PR 481]: https://github.com/envoyproxy/gateway/pull/481 +[Pull Request]: https://github.com/envoyproxy/gateway/pulls +[Quickstart Guide]: https://github.com/envoyproxy/gateway/blob/main/docs/user/QUICKSTART.md +[release GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/release.yaml +[release workflow]: https://github.com/envoyproxy/gateway/actions/workflows/release.yaml +[image]: https://hub.docker.com/r/envoyproxy/gateway/tags +[release]: https://github.com/envoyproxy/gateway/releases +[Generate]: https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes From d8bb99bb311d6a89131c8170f53827a29d286c25 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Mon, 17 Oct 2022 10:27:00 -0600 Subject: [PATCH 026/113] use RefNotPermitted reason for invalid cross-namespace TLS cert ref (#580) * use RefNotPermitted reason for invalid cross-namespace TLS cert ref Closes #538. Signed-off-by: Steve Kriss --- ...valid-tls-configuration-secret-in-other-namespace.out.yaml | 2 +- internal/gatewayapi/translator.go | 2 +- test/conformance/conformance_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml index 4dd99085ff9..757e4c695e9 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml @@ -28,7 +28,7 @@ gateways: conditions: - type: ResolvedRefs status: "False" - reason: InvalidCertificateRef + reason: RefNotPermitted message: Certificate ref to secret default/tls-secret-1 not permitted by any ReferenceGrant - type: Ready status: "False" diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 0d4db0be7e0..80ed034e8ba 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -431,7 +431,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap listener.SetCondition( v1beta1.ListenerConditionResolvedRefs, metav1.ConditionFalse, - v1beta1.ListenerReasonInvalidCertificateRef, + v1beta1.ListenerReasonRefNotPermitted, fmt.Sprintf("Certificate ref to secret %s/%s not permitted by any ReferenceGrant", *certificateRef.Namespace, certificateRef.Name), ) break diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index 9490e96bf5a..b50613936b9 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -46,7 +46,7 @@ func TestGatewayAPIConformance(t *testing.T) { Debug: *flags.ShowDebug, CleanupBaseResources: *flags.CleanupBaseResources, ValidUniqueListenerPorts: validUniqueListenerPorts, - SupportedFeatures: []suite.SupportedFeature{suite.SupportReferenceGrant}, + SupportedFeatures: []suite.SupportedFeature{suite.SupportReferenceGrant}, }) cSuite.Setup(t) egTests := []suite.ConformanceTest{ @@ -63,7 +63,7 @@ func TestGatewayAPIConformance(t *testing.T) { tests.HTTPRouteInvalidCrossNamespaceBackendRef, tests.GatewaySecretReferenceGrantAllInNamespace, tests.GatewaySecretReferenceGrantSpecific, - // Uncomment when https://github.com/envoyproxy/gateway/issues/538 is fixed. + // Uncomment when https://github.com/envoyproxy/gateway/issues/539 is fixed. /*tests.GatewaySecretMissingReferenceGrant, tests.GatewaySecretInvalidReferenceGrant,*/ } From 0cf2f878799d9b5fd223d0394687fc91b19ebffa Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 18 Oct 2022 00:30:33 +0800 Subject: [PATCH 027/113] feat: support markdown resources (#571) * feat: support markdown resources Signed-off-by: bitliu * update Signed-off-by: bitliu Signed-off-by: bitliu --- docs/conf.py | 6 ++ docs/design/ROADMAP.md | 9 ++- docs/design/{CONFIG_API.md => config-api.md} | 44 +++++++++----- docs/design/gatewayapi-translator.md | 10 +++- .../{SYSTEM_DESIGN.md => system-design.md} | 60 ++++++++++++------- docs/design/watching.md | 8 +-- docs/index.rst | 6 ++ docs/user/QUICKSTART.md | 28 ++++++++- tools/src/sphinx-build/requirements.txt | 1 + 9 files changed, 125 insertions(+), 47 deletions(-) rename docs/design/{CONFIG_API.md => config-api.md} (95%) rename docs/design/{SYSTEM_DESIGN.md => system-design.md} (95%) diff --git a/docs/conf.py b/docs/conf.py index f38797e5714..6f6d6845e25 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,6 +7,7 @@ extensions = [ 'sphinx.ext.duration', 'sphinx.ext.autosectionlabel', + 'myst_parser', ] html_theme = 'alabaster' @@ -36,6 +37,11 @@ envoyVersion = os.environ["ENVOY_VERSION"] gatewayAPIVersion = os.environ["GATEWAYAPI_VERSION"] +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + variables_to_export = [ "envoyVersion", "gatewayAPIVersion", diff --git a/docs/design/ROADMAP.md b/docs/design/ROADMAP.md index 1d5d942d83e..d6ec649e4a2 100644 --- a/docs/design/ROADMAP.md +++ b/docs/design/ROADMAP.md @@ -1,4 +1,4 @@ -## Introduction +# Roadmap This document serves as a high-level reference for Envoy Gateway users and contributors to understand the direction of the project. @@ -16,7 +16,8 @@ the project. If you don't know where to start contributing, help is needed to reduce technical, automation, and documentation debt. Look for issues with the `help wanted` label to get started. -## Roadmap +## Details + Roadmap features and timelines may change based on feedback, community contributions, etc. If you depend on a specific roadmap item, you're encouraged to attend a community meeting to discuss the details, or help us deliver the feature by contributing to the project. @@ -24,6 +25,7 @@ contributing to the project. `Last Updated: October 2022` ### [v0.2.0][v0.2.0]: Establish a Solid Foundation + - Complete the core Envoy Gateway implementation- [Issue #60][60]. - Establish initial testing, e2e, integration, etc- [Issue #64][64]. - Establish user and developer project documentation- [Issue #17][17]. @@ -31,17 +33,18 @@ contributing to the project. - Setup a CI/CD pipeline- [Issue #63][63]. ### [v0.3.0][v0.3.0]: Drive Advanced Features through Extension Mechanisms + - Global Rate Limiting - AuthN/AuthZ- [Issue #336][336]. - Lets Encrypt Integration ### [v0.4.0][v0.4.0]: Manageability and Scale + - Tooling for devs/infra admins to aid in managing/maintaining EG - Support advanced provisioning use cases (e.g. multi-cluster, serverless, etc.) - Perf testing (EG specifically) - Support for Chaos engineering? -[eg_board]: https://github.com/orgs/envoyproxy/projects/1/views/1?layout=board [issue]: https://github.com/envoyproxy/gateway/issues [meeting]: https://docs.google.com/document/d/1leqwsHX8N-XxNEyTflYjRur462ukFxd19Rnk3Uzy55I/edit?usp=sharing [pr]: https://github.com/envoyproxy/gateway/compare diff --git a/docs/design/CONFIG_API.md b/docs/design/config-api.md similarity index 95% rename from docs/design/CONFIG_API.md rename to docs/design/config-api.md index fbbfb2aa494..5a8478d8df8 100644 --- a/docs/design/CONFIG_API.md +++ b/docs/design/config-api.md @@ -1,28 +1,19 @@ -Configuration API Design -=================== - -# Table of Contents -1. [Motivation](#motivation) -2. [Goals](#goals) -3. [Non-Goals](#non-goals) -4. [Control Plane API](#control_plane_api) - 1. [Gateway Type](#gateway) - 2. [Provider Type](#provider) - 3. [Configuration Examples](#control_plane_configuration) -6. [Data Plane API](#data_plane_api) - 1. [Configuration Examples](#data_plane_configuration) +# Configuration API Design ## Motivation + [Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway. The control plane is configured statically at startup and the data plane is configured dynamically through Kubernetes resources, primarily [Gateway API][gw_api] objects. Refer to the Envoy Gateway [design doc][design_doc] for additional details regarding Envoy Gateway terminology and configuration. ## Goals + * Define an __initial__ API to configure Envoy Gateway at startup. * Define an __initial__ API for configuring the managed data plane, e.g. Envoy proxies. ## Non-Goals + * Implementation of the configuration APIs. * Define the `status` subresource of the configuration APIs. * Define a __complete__ set of APIs for configuring Envoy Gateway. As stated in the [Goals](#goals), this document @@ -32,6 +23,7 @@ Envoy Gateway terminology and configuration. * Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. ## Control Plane API + The `EnvoyGateway` API defines the control plane configuration, e.g. Envoy Gateway. Key points of this API are: * It will define Envoy Gateway's startup configuration file. If the file does not exist, Envoy Gateway will start up @@ -44,6 +36,7 @@ The `EnvoyGateway` API defines the control plane configuration, e.g. Envoy Gatew * If data plane static configuration is required in the future, Envoy Gateway will use a separate file for this purpose. The `v1alpha1` version and `config.gateway.envoyproxy.io` API group get generated: + ```go // gateway/api/config/v1alpha1/doc.go @@ -54,6 +47,7 @@ package v1alpha1 ``` The initial `EnvoyGateway` API being proposed: + ```go // gateway/api/config/v1alpha1/envoygateway.go @@ -139,30 +133,37 @@ type FileProvider struct { // TODO: Add config as use cases are better understood. } ``` + __Note:__ Provider-specific configuration is defined in the `{$PROVIDER_NAME}Provider` API. ### Gateway + Gateway defines desired configuration of [Gateway API][gw_api] controllers that reconcile and translate Gateway API resources into the Intermediate Representation (IR). Refer to the Envoy Gateway [design doc][design_doc] for additional details. ### Provider + Provider defines the desired configuration of an Envoy Gateway provider. A provider is an infrastructure component that Envoy Gateway calls to establish its runtime configuration. Provider is a [union type][union]. Therefore, Envoy Gateway can be configured with only one provider based on the `type` discriminator field. Refer to the Envoy Gateway [design doc][design_doc] for additional details. ### Control Plane Configuration + The configuration file is defined by the EnvoyGateway API type. At startup, Envoy Gateway searches for the configuration at "/etc/envoy-gateway/config.yaml". Start Envoy Gateway: + ```shell $ ./envoy-gateway ``` + Since the configuration file does not exist, Envoy Gateway will start with default configuration parameters. The Kubernetes provider can be configured explicitly using `provider.kubernetes`: + ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml apiVersion: config.gateway.envoyproxy.io/v1alpha1 @@ -172,9 +173,11 @@ provider: kubernetes: {} EOF ``` + This configuration will cause Envoy Gateway to use the Kubernetes provider with default configuration parameters. The Kubernetes provider can be configured using the `provider` field. For example, the `foo` field can be set to "bar": + ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml apiVersion: config.gateway.envoyproxy.io/v1alpha1 @@ -185,11 +188,13 @@ provider: foo: bar EOF ``` + __Note:__ The Provider API from the Kubernetes package is currently undefined and `foo: bar` is provided for illustration purposes only. The same API structure is followed for each supported provider. The following example causes Envoy Gateway to use the File provider: + ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml apiVersion: config.gateway.envoyproxy.io/v1alpha1 @@ -200,12 +205,14 @@ provider: foo: bar EOF ``` + __Note:__ The Provider API from the File package is currently undefined and `foo: bar` is provided for illustration purposes only. Gateway API-related configuration is expressed through the `gateway` field. If unspecified, Envoy Gateway will use default configuration parameters for `gateway`. The following example causes the [GatewayClass][gc] controller to manage GatewayClasses with controllerName `foo` instead of the default `gateway.envoyproxy.io/gatewayclass-controller`: + ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml apiVersion: config.gateway.envoyproxy.io/v1alpha1 @@ -215,11 +222,13 @@ gateway: ``` With any of the above configuration examples, Envoy Gateway can be started without any additional arguments: + ```shell $ ./envoy-gateway ``` ## Data Plane API + The data plane is configured dynamically through Kubernetes resources, primarily [Gateway API][gw_api] objects. Optionally, the data plane infrastructure can be configured by referencing a [custom resource (CR)][cr] through `spec.parametersRef` of the managed GatewayClass. The `envoyproxies` API defines the data plane infrastructure @@ -233,6 +242,7 @@ configuration and is represented as the CR referenced by the managed GatewayClas > not propagated down to existing Gateways. The initial `envoyproxies` API being proposed: + ```go // gateway/api/config/v1alpha1/envoyproxy.go @@ -262,12 +272,15 @@ type EnvoyProxyStatus struct { // Undefined by this design spec. } ``` + The EnvoyProxySpec and EnvoyProxyStatus fields will be defined in the future as proxy infrastructure configuration use cases are better understood. ### Data Plane Configuration + GatewayClass and Gateway resources define the data plane infrastructure. Note that all examples assume Envoy Gateway is running with the Kubernetes provider. + ```yaml apiVersion: gateway.networking.k8s.io/v1beta1 kind: GatewayClass @@ -287,12 +300,14 @@ spec: protocol: HTTP port: 80 ``` + Since the GatewayClass does not define `spec.parametersRef`, the data plane is provisioned using default configuration parameters. All Envoy proxies will be configured with a http listener and a Kubernetes LoadBalancer service listening on port 80. The following example will configure the data plane to use a ClusterIP service instead of the default LoadBalancer service: + ```yaml apiVersion: gateway.networking.k8s.io/v1beta1 kind: GatewayClass @@ -324,13 +339,12 @@ spec: networkPublishing: type: ClusterIPService ``` + __Note:__ The NetworkPublishing API is currently undefined and is provided here for illustration purposes only. [issue_51]: https://github.com/envoyproxy/gateway/issues/51 [design_doc]: https://github.com/envoyproxy/gateway/blob/main/docs/design/SYSTEM_DESIGN.md -[xds]: https://github.com/cncf/xds [gw_api]: https://gateway-api.sigs.k8s.io/ -[config_guide]: https://github.com/envoyproxy/gateway/blob/main/docs/CONFIG.md [gc]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass [cr]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ [union]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#unions diff --git a/docs/design/gatewayapi-translator.md b/docs/design/gatewayapi-translator.md index b79abc314ea..5f2d4abcffa 100644 --- a/docs/design/gatewayapi-translator.md +++ b/docs/design/gatewayapi-translator.md @@ -1,22 +1,27 @@ # Gateway API Translator Design ## Assumptions + - initially target core conformance features only, to be followed by extended conformance features ## Inputs and Outputs The main inputs to the Gateway API translator are: + - the GatewayClass to process - Gateways, HTTPRoutes, Services, Secrets The outputs of the Gateway API translator are: + - IR - status updates for GatewayClass, Gateways, HTTPRoutes ## Listener Compatibility + Since Envoy Gateway handles all Gateways for a given GatewayClass, we need to determine the compatibility of _all_ Listeners across _all_ of those Gateways. The rules are: + - for a given port number, every Listener using that port number must have a compatible protocol (either all HTTP, or all HTTPS/TLS). - for a given port number, every Listener using that port number must have a distinct hostname (at most one Listener per port can have no hostname). @@ -60,7 +65,7 @@ spec: hostname: whales.envoygateway.io ``` -#### Example 2: Gateways with compatible Listeners (same port & protocol, one hostname specified, one not) +#### Example 2: Gateways with compatible Listeners (same port & protocol, one hostname specified, one not) ```yaml kind: Gateway @@ -171,6 +176,7 @@ Gateway API specifies a rich set of status fields & conditions for each resource To be conformant, Envoy Gateway needs to compute the appropriate status fields and conditions as it's processing resources. Status needs to be computed and set for: + - the GatewayClass (gatewayclass.status.conditions) - each Listener for each Gateway (gateway.status.listeners) - each Gateway, based on its Listeners' statuses (gateway.status.conditions) @@ -183,7 +189,6 @@ The Gateway API translator will take the approach of populating status on the re The following roughly outlines the translation process. Each step may produce (1) IR; and (2) status updates on Gateway API resources. -``` 1. Process Gateway Listeners - validate unique hostnames/ports/protcols - validate/compute supported kinds @@ -208,7 +213,6 @@ Each step may produce (1) IR; and (2) status updates on Gateway API resources. - foreach matching listener: - foreach hostname intersection with route: - add each computed route rule to host -``` ## Context Structs diff --git a/docs/design/SYSTEM_DESIGN.md b/docs/design/system-design.md similarity index 95% rename from docs/design/SYSTEM_DESIGN.md rename to docs/design/system-design.md index 2b741bc110c..1f30dc7203e 100644 --- a/docs/design/SYSTEM_DESIGN.md +++ b/docs/design/system-design.md @@ -1,33 +1,41 @@ -## System Design +# System Design + +## Goals -### Goals * Define the system components needed to satisfy the requirements of Envoy Gateway. -### Non-Goals +## Non-Goals + * Create a detailed design and interface specification for each system component. -### Terminology +## Terminology + * Control Plane- A collection of inter-related software components for providing application gateway and routing functionality. The control plane is implemented by Envoy Gateway and provides services for managing the data plane. These services are detailed in the [components](#components) section. * Data Plane- Provides intelligent application-level traffic routing and is implemented as one or more Envoy proxies. -### Architecture +## Architecture + ![Architecture](../images/architecture.png) -### Configuration +## Configuration + Envoy Gateway is configured statically at startup and the managed data plane is configured dynamically through Kubernetes resources, primarily [Gateway API][gw_api] objects. -#### Static Configuration +### Static Configuration + Static configuration is used to configure Envoy Gateway at startup, i.e. change the GatewayClass controllerName, configure a Provider, etc. Currently, Envoy Gateway only supports configuration through a configuration file. If the configuration file is not provided, Envoy Gateway will start up with default configuration parameters. -#### Dynamic Configuration +### Dynamic Configuration + Dynamic configuration is based on the concept of a declaring the desired state of the data plane and using reconciliation loops to drive the actual state toward the desired state. The desired state of the data plane is defined as Kubernetes resources that provide the following services: + * Infrastructure Management- Manage the data plane infrastructure, i.e. deploy, upgrade, etc. This configuration is expressed through [GatewayClass][gc] and [Gateway][gw] resources. A TBD [Custom Resource][cr] can be referenced by `gatewayclass.spec.parametersRef` to modify data plane infrastructure default parameters, @@ -38,58 +46,68 @@ defined as Kubernetes resources that provide the following services: Although a backend can be any valid Kubernetes Group/Kind resource, Envoy Gateway only supports a [Service][svc] reference. -### Components +## Components Envoy Gateway is made up of several components that communicate in-process; how this communication happens is described in [watching.md][]. -#### Provider +### Provider + A Provider is an infrastructure component that Envoy Gateway calls to establish its runtime configuration, resolve services, persist data, etc. Kubernetes and File are the only supported providers. However, other providers can be added in the future as Envoy Gateway use cases are better understood. A provider is configured at start up through Envoy Gateway's [static configuration](#static-configuration). -##### Kubernetes Provider +#### Kubernetes Provider + * Uses Kubernetes-style controllers to reconcile Kubernetes resources that comprise the [dynamic configuration](#dynamic-configuration). * Manages the data plane through Kubernetes API CRUD operations. * Uses Kubernetes for Service discovery. * Uses etcd (via Kubernetes API) to persist data. -##### File Provider +#### File Provider * Uses a file watcher to watch files in a directory that define the data plane configuration. * Manages the data plane by calling internal APIs, e.g. `CreateDataPlane()`. * Uses the host's DNS for Service discovery. * If needed, the local filesystem is used to persist data. -#### Resource Watcher +### Resource Watcher + The Resource Watcher watches resources used to establish and maintain Envoy Gateway's dynamic configuration. The mechanics for watching resources is provider-specific, e.g. informers, caches, etc. are used for the Kubernetes provider. The Resource Watcher uses the configured provider for input and provides resources to the Resource Translator as output. -#### Resource Translator +### Resource Translator + The Resource Translator translates external resources, e.g. GatewayClass, from the Resource Watcher to the Intermediate Representation (IR). It is responsible for: + * Translating infrastructure-specific resources/fields from the Resource Watcher to the Infra IR. * Translating proxy configuration resources/fields from the Resource Watcher to the xDS IR. -#### Intermediate Representation (IR) +### Intermediate Representation (IR) + The Intermediate Representation defines internal data models that external resources are translated into. This allows Envoy Gateway to be decoupled from the external resources used for dynamic configuration. The IR consists of an Infra IR used as input for the Infra Manager and an xDS IR used as input for the xDS Translator. + * Infra IR- Used as the internal definition of the managed data plane infrastructure. * xDS IR- Used as the internal definition of the managed data plane xDS configuration. -#### xDS Translator +### xDS Translator + The xDS Translator translates the xDS IR into xDS Resources that are consumed by the xDS server. -#### xDS Server +### xDS Server + The xDS Server is a xDS gRPC Server based on [Go Control Plane][go_cp]. Go Control Plane implements the xDS Server Protocol and is responsible for using xDS to configure the data plane. -#### Infra Manager +### Infra Manager + The Infra Manager is a provider-specific component responsible for managing the following infrastructure: * Data Plane - Manages all the infrastructure required to run the managed Envoy proxies. For example, CRUD Deployment, @@ -101,7 +119,8 @@ The Infra Manager is a provider-specific component responsible for managing the The Infra Manager consumes the Infra IR as input to manage the data plane infrastructure. -### Design Decisions +## Design Decisions + * Envoy Gateway will consume one [GatewayClass][gc] by comparing its configured controller name with `spec.controllerName` of a GatewayClass. If multiple GatewayClasses exist with the same `spec.controllerName`, Envoy Gateway will follow Gateway API [guidelines][gwapi_conflicts] to resolve the conflict. @@ -119,7 +138,8 @@ The Infra Manager consumes the Infra IR as input to manage the data plane infras The draft for this document is [here][draft_design]. -### Caveats +## Caveats + * The custom resource used to configure the data plane infrastructure is TBD. Track [issue 95][issue_95] for the latest updates. * Envoy Gateway's static configuration spec is currently undefined. Track [issue 95][issue_95] for the latest updates. diff --git a/docs/design/watching.md b/docs/design/watching.md index e505b6280a0..35b45abe622 100644 --- a/docs/design/watching.md +++ b/docs/design/watching.md @@ -1,4 +1,4 @@ -## Watching the results of other components +# Watching Components Design Envoy Gateway is made up of several components that communicate in-process. Some of them (namely providers) watch external resources, and "publish" what they see for other components to consume; others watch what another publishes and @@ -9,7 +9,7 @@ To facilitate this communication use the [watchable][] library. The `watchable. standard library's `sync.Map` type, but supports a `.Subscribe` (and `.SubscribeSubset`) method that promotes a pub/sub pattern. -### pub +## Pub Many of the things we communicate around are naturally named, either by a bare "name" string or by a "name"/"namespace" tuple. And because `watchable.Map` is typed, it makes sense to have one map for each type of thing (very similar to if @@ -34,7 +34,7 @@ method) the current value is a no-op; it won't trigger an event for subscribers. doesn't have as much state to keep track of; it doesn't need to know "did I already publish this thing", it can just `.Store` its data and `watchable` will do the right thing. -### sub +## Sub Meanwhile, the translator and other interested components subscribe to it with `table.Thing.Subscribe` (or `table.Thing.SubscribeSubset` if they only care about a few "Thing"s). So the translator goroutine might look like: @@ -102,7 +102,7 @@ entries for those pre-existing items; if you are working with `snapshot.Update` must add special handling for your first read. We have a utility function `./internal/message.HandleSubscription` to help with this. -### other notes +## Other Notes The common pattern will likely be that the entrypoint that launches the goroutines for each component instantiates the map, and passes them to the appropriate publishers and subscribers; same as if they were communicating via a dumb diff --git a/docs/index.rst b/docs/index.rst index 4304b533b8a..ec7467ae4c1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,5 +20,11 @@ standalone or Kubernetes-based application gateway. intro/index intro/compatibility + user/QUICKSTART + design/system-design + design/ROADMAP + design/gatewayapi-translator + design/watching + design/config-api about_docs get_involved diff --git a/docs/user/QUICKSTART.md b/docs/user/QUICKSTART.md index 980d1e371cd..931ca88bd6c 100644 --- a/docs/user/QUICKSTART.md +++ b/docs/user/QUICKSTART.md @@ -1,57 +1,72 @@ -## Introduction +# Quickstart + This guide will help you get started with Envoy Gateway in a few simple steps. ## Prerequisites + A Kubernetes cluster. __Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. ## Installation + Install the Gateway API CRDs: + ```shell kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/gatewayapi-crds.yaml ``` Run Envoy Gateway: + ```shell kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/install.yaml ``` Run the example app: + ```shell kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/httpbin.yaml ``` The Gateway API resources must be created in the following order. First, create the GatewayClass: + ```shell kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/gatewayclass.yaml ``` Create the Gateway: + ```shell kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/gateway.yaml ``` Create the HTTPRoute to route traffic through Envoy proxy to the example app: + ```shell kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/httproute.yaml ``` ### Testing the configuration + Port forward to the Envoy service: + ```shell kubectl -n envoy-gateway-system port-forward service/envoy-default-eg 8888:8080 & ``` Curl the example app through Envoy proxy: + ```shell curl --verbose --header "Host: www.example.com" http://localhost:8888/get ``` + You can replace `get` with any of the supported [httpbin methods][httpbin_methods]. ### For clusters with External Loadbalancer support + You can also test the same functionality by sending traffic to the External IP. To get the external IP of the Envoy service, run: + ```shell export GATEWAY_HOST=$(kubectl get svc/envoy-default-eg -n envoy-gateway-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') ``` @@ -60,46 +75,55 @@ In certain environments, the load balancer may be exposed using a hostname, inst `ip` in the above command with `hostname`. Curl the example app through Envoy proxy: + ```shell curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST:8080/get ``` + You can replace `get` with any of the supported [httpbin methods][httpbin_methods]. ## Clean-Up + Use the steps in this section to uninstall everything from the quickstart guide. Delete the HTTPRoute: + ```shell kubectl delete httproute/httpbin ``` Delete the Gateway: + ```shell kubectl delete gateway/eg ``` Delete the GatewayClass: + ```shell kubectl delete gc/eg ``` Uninstall the example app: + ```shell kubectl delete -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/httpbin.yaml ``` Uninstall Envoy Gateway: + ```shell kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/install.yaml ``` Uninstall Gateway API CRDs: + ```shell kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/gatewayapi-crds.yaml ``` ## Next Steps + Checkout the [Developer Guide](../../DEVELOPER.md) to get involved in the project. -[kind]: https://kind.sigs.k8s.io/ [httpbin_methods]: https://httpbin.org/#/HTTP_Methods diff --git a/tools/src/sphinx-build/requirements.txt b/tools/src/sphinx-build/requirements.txt index baa05d40d5c..de4d2c42f4e 100644 --- a/tools/src/sphinx-build/requirements.txt +++ b/tools/src/sphinx-build/requirements.txt @@ -1 +1,2 @@ Sphinx==5.1.1 +myst-parser==0.18.1 \ No newline at end of file From 38455832eaeece6196ddd36ce9e9980c0b14e6eb Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 09:41:15 -0700 Subject: [PATCH 028/113] Moves Changelogs to Release Notes (#546) * Moves Changelogs to Release Notes Signed-off-by: danehans * Adds release-artifacts target with release notes Signed-off-by: danehans Signed-off-by: danehans --- .github/workflows/release.yaml | 5 +++-- changelogs/0.1.0.yaml => release-notes/v0.1.0.yaml | 0 changelogs/0.2.0-rc1.yaml => release-notes/v0.2.0-rc1.yaml | 0 changelogs/0.2.0-rc2.yaml => release-notes/v0.2.0-rc2.yaml | 0 tools/make/kube.mk | 5 +++++ 5 files changed, 8 insertions(+), 2 deletions(-) rename changelogs/0.1.0.yaml => release-notes/v0.1.0.yaml (100%) rename changelogs/0.2.0-rc1.yaml => release-notes/v0.2.0-rc1.yaml (100%) rename changelogs/0.2.0-rc2.yaml => release-notes/v0.2.0-rc2.yaml (100%) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7b1e345892b..efcc8ade072 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,8 +27,8 @@ jobs: run: | skopeo copy --all docker://docker.io/envoyproxy/gateway-dev:${{ env.sha_short }} docker://docker.io/envoyproxy/gateway:${{ env.release_tag }} - - name: Generate Release Manifests - run: make generate-manifests IMAGE=envoyproxy/gateway TAG=${{ env.release_tag}} OUTPUT_DIR=release-artifacts + - name: Generate Release Artifacts + run: make generate-artifacts IMAGE=envoyproxy/gateway TAG=${{ env.release_tag}} OUTPUT_DIR=release-artifacts - name: Upload Release Manifests uses: softprops/action-gh-release@v1 @@ -36,3 +36,4 @@ jobs: files: | release-artifacts/install.yaml release-artifacts/quickstart.yaml + release-artifacts/release-notes.yaml diff --git a/changelogs/0.1.0.yaml b/release-notes/v0.1.0.yaml similarity index 100% rename from changelogs/0.1.0.yaml rename to release-notes/v0.1.0.yaml diff --git a/changelogs/0.2.0-rc1.yaml b/release-notes/v0.2.0-rc1.yaml similarity index 100% rename from changelogs/0.2.0-rc1.yaml rename to release-notes/v0.2.0-rc1.yaml diff --git a/changelogs/0.2.0-rc2.yaml b/release-notes/v0.2.0-rc2.yaml similarity index 100% rename from changelogs/0.2.0-rc2.yaml rename to release-notes/v0.2.0-rc2.yaml diff --git a/tools/make/kube.mk b/tools/make/kube.mk index e0c8d3dd4b0..4025dfbfe76 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -106,3 +106,8 @@ generate-manifests: $(tools/kustomize) ## Generate Kubernetes release manifests. @echo "\033[36m===========> Added: $(OUTPUT_DIR)/install.yaml\033[0m" cp examples/kubernetes/quickstart.yaml $(OUTPUT_DIR)/quickstart.yaml @echo "\033[36m===========> Added: $(OUTPUT_DIR)/quickstart.yaml\033[0m" + +.PHONY: generate-artifacts +generate-artifacts: generate-manifests ## Generate release artifacts. + cp -r $(ROOT_DIR)/release-notes/$(TAG).yaml $(OUTPUT_DIR)/release-notes.yaml + @echo "\033[36m===========> Added: $(OUTPUT_DIR)/release-notes.yaml\033[0m" From 5e3db60596491c8bb5410ffdb65fb634f65282c6 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 10:07:17 -0700 Subject: [PATCH 029/113] Adds HTTPRouting User Doc (#558) Signed-off-by: danehans Signed-off-by: danehans --- docs/user/HTTP_ROUTING.md | 98 ++++++++++ examples/kubernetes/http-routing.yaml | 272 ++++++++++++++++++++++++++ 2 files changed, 370 insertions(+) create mode 100644 docs/user/HTTP_ROUTING.md create mode 100644 examples/kubernetes/http-routing.yaml diff --git a/docs/user/HTTP_ROUTING.md b/docs/user/HTTP_ROUTING.md new file mode 100644 index 00000000000..004122bcb4b --- /dev/null +++ b/docs/user/HTTP_ROUTING.md @@ -0,0 +1,98 @@ +# HTTP Routing + +The [HTTPRoute][] resource allows users to configure HTTP routing by matching HTTP traffic and forwarding it to +Kubernetes backends. Currently, the only supported backend supported by Envoy Gateway is a Service resource. This guide +shows how to route traffic based on host, header, and path fields and forward the traffic to different Kubernetes +Services. To learn more about HTTP routing, refer to the [Gateway API documentation][]. + +Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and then install the example +resources used for this guide. +```shell +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0/examples/kubernetes/http-routing.yaml +``` +The manifest installs a [GatewayClass][], [Gateway][], four Deployments, four Services, and three HTTPRoute resources. + +The GatewayClass is a cluster-scoped resource that represents a class of Gateways that can be instantiated. Envoy +Gateway is configured by default to manage GatewayClasses with +`controllerName: gateway.envoyproxy.io/gatewayclass-controller`. + +Check the status of the GatewayClass: +```shell +kubectl get gc --selector=example=http-routing +``` +The status should reflect "Accepted=True", indicating Envoy Gateway is managing the GatewayClass. + +A Gateway represents configuration of infrastructure. When a Gateway is created, [Envoy proxy][] infrastructure is +provisioned or configured by Envoy Gateway. The `gatewayClassName` defines the name of a GatewayClass used by this +Gateway. Check the status of the Gateway: +```shell +kubectl get gateways --selector=example=http-routing +``` +The status should reflect "Ready=True", indicating the Envoy proxy infrastructure has been provisioned. The status also +provides the address of the Gateway. This address is used later in the guide to test connectivity to proxied backend +services. + +The three HTTPRoute resources create routing rules on the Gateway. In order to receive traffic from a Gateway, +an HTTPRoute must be configured with `parentRefs` which reference the parent Gateway(s) that it should be attached to. +An HTTPRoute can match against a [single set of hostnames][spec]. These hostnames are matched before any other matching +within the HTTPRoute takes place. Since `example.com`, `foo.example.com`, and `bar.example.com` are separate hosts with +different routing requirements, each is deployed as its own HTTPRoute - `example-route, ``foo-route`, and `bar-route`. + +Check the status of the HTTPRoutes: +```shell +kubectl get httproutes --selector=example=http-routing -o yaml +``` +The status for each HTTPRoute should surface "Accepted=True" and a `parentRef` that references the example Gateway. +The `example-route` matches any traffic for "example.com" and forwards it to the "example-svc" Service. Before testing +HTTP routing to the `example-svc` backend, get the Gateway's address. +```shell +export GATEWAY_HOST=$(kubectl get gateway/example-gateway -o jsonpath='{.status.addresses[0].value}') +``` + +Test HTTP routing to the `example-svc` backend. +```shell +curl -vvv --header "Host: example.com" "http://${GATEWAY_HOST}/" +``` +A `200` status code should be returned and the body should include `"pod": "example-backend-*"` indicating the traffic +was routed to the example backend service. If you change the hostname to a hostname not represented in any of the +HTTPRoutes, e.g. "www.example.com", the HTTP traffic will not be routed and a `404` should be returned. + +The `foo-route` matches any traffic for `foo.example.com` and applies its routing rules to forward the traffic to the +"foo-svc" Service. Since there is only one path prefix match for `/login`, only `foo.example.com/login/*` traffic will +be forwarded. Test HTTP routing to the `foo-svc` backend. +```shell +curl -vvv --header "Host: foo.example.com" "http://${GATEWAY_HOST}/login" +``` + +A `200` status code should be returned and the body should include `"pod": "foo-backend-*"` indicating the traffic +was routed to the foo backend service. Traffic to any other paths that do not begin with `/login` will not be matched by +this HTTPRoute. Test this by removing `/login` from the request. +```shell +curl -vvv --header "Host: foo.example.com" "http://${GATEWAY_HOST}/" +``` +The HTTP traffic will not be routed and a `404` should be returned. + +Similarly, the `bar-route` HTTPRoute matches traffic for `bar.example.com`. All traffic for this hostname will be +evaluated against the routing rules. The most specific match will take precedence which means that any traffic with the +`env:canary` header will be forwarded to `bar-svc-canary` and if the header is missing or not `canary` then it'll be +forwarded to `bar-svc`. Test HTTP routing to the `bar-svc` backend. +```shell +curl -vvv --header "Host: bar.example.com" "http://${GATEWAY_HOST}/" +``` +A `200` status code should be returned and the body should include `"pod": "bar-backend-*"` indicating the traffic +was routed to the foo backend service. + +Test HTTP routing to the `bar-canary-svc` backend by adding the `env: canary` header to the request. +```shell +curl -vvv --header "Host: bar.example.com" --header "env: canary" "http://${GATEWAY_HOST}/" +``` +A `200` status code should be returned and the body should include `"pod": "bar-canary-backend-*"` indicating the +traffic was routed to the foo backend service. + +[HTTPRoute]: https://gateway-api.sigs.k8s.io/api-types/httproute/ +[Gateway API documentation]: https://gateway-api.sigs.k8s.io/ +[GatewayClass]: https://gateway-api.sigs.k8s.io/api-types/gatewayclass/ +[Gateway]: https://gateway-api.sigs.k8s.io/api-types/gateway/ +[Envoy proxy]: https://www.envoyproxy.io/ +[spec]: /references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteSpec +[svc]:https://kubernetes.io/docs/concepts/services-networking/service/ diff --git a/examples/kubernetes/http-routing.yaml b/examples/kubernetes/http-routing.yaml new file mode 100644 index 00000000000..2dc774b715a --- /dev/null +++ b/examples/kubernetes/http-routing.yaml @@ -0,0 +1,272 @@ +# This file contains the base resources that the docs/user/HTTP_ROUTING.md guide relies on. +# This includes a GatewayClass, Gateway, Services and Deployments that are used as backends +# for routing traffic. +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: example-gateway-class + labels: + example: http-routing +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: example-gateway + labels: + example: http-routing +spec: + gatewayClassName: example-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: example-svc + labels: + example: http-routing +spec: + ports: + - name: http + port: 8080 + targetPort: 3000 + selector: + app: example-backend +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-backend + labels: + app: example-backend + example: http-routing +spec: + replicas: 1 + selector: + matchLabels: + app: example-backend + template: + metadata: + labels: + app: example-backend + spec: + containers: + - name: example-backend + image: gcr.io/k8s-staging-ingressconformance/echoserver:v20220815-e21d1a4 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: example-route + labels: + example: http-routing +spec: + parentRefs: + - name: example-gateway + hostnames: + - "example.com" + rules: + - backendRefs: + - name: example-svc + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: foo-svc + labels: + example: http-routing +spec: + ports: + - name: http + port: 8080 + targetPort: 3000 + selector: + app: foo-backend +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo-backend + labels: + app: foo-backend + example: http-routing +spec: + replicas: 1 + selector: + matchLabels: + app: foo-backend + template: + metadata: + labels: + app: foo-backend + spec: + containers: + - name: foo-backend + image: gcr.io/k8s-staging-ingressconformance/echoserver:v20220815-e21d1a4 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: foo-route + labels: + example: http-routing +spec: + parentRefs: + - name: example-gateway + hostnames: + - "foo.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: /login + backendRefs: + - name: foo-svc + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: bar-svc + labels: + example: http-routing +spec: + ports: + - name: http + port: 8080 + targetPort: 3000 + selector: + app: bar-backend +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bar-backend + labels: + app: bar-backend + example: http-routing +spec: + replicas: 1 + selector: + matchLabels: + app: bar-backend + template: + metadata: + labels: + app: bar-backend + spec: + containers: + - name: bar-backend + image: gcr.io/k8s-staging-ingressconformance/echoserver:v20220815-e21d1a4 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Service +metadata: + name: bar-canary-svc + labels: + example: http-routing +spec: + ports: + - name: http + port: 8080 + targetPort: 3000 + selector: + app: bar-canary-backend +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bar-canary-backend + labels: + app: bar-canary-backend + example: http-routing +spec: + replicas: 1 + selector: + matchLabels: + app: bar-canary-backend + template: + metadata: + labels: + app: bar-canary-backend + spec: + containers: + - name: bar-canary-backend + image: gcr.io/k8s-staging-ingressconformance/echoserver:v20220815-e21d1a4 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: bar-route + labels: + example: http-routing +spec: + parentRefs: + - name: example-gateway + hostnames: + - "bar.example.com" + rules: + - matches: + - headers: + - type: Exact + name: env + value: canary + backendRefs: + - name: bar-canary-svc + port: 8080 + - backendRefs: + - name: bar-svc + port: 8080 From 1caaa48a07a464ec1987a06da548cca9bebbe25a Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 10:13:30 -0700 Subject: [PATCH 030/113] Updates Dev Doc to Use Gateway Ns/Name (#561) Signed-off-by: danehans Signed-off-by: danehans --- DEVELOPER.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index fb5639e483c..c669eeedce1 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -74,9 +74,9 @@ and runs the Gateway API conformance tests. ### Debugging the Envoy Config An easy way to view the envoy config that Envoy Gateway is using is to port-forward to the admin interface port (currently `19000`) -on the Envoy Gateway deployment so that it can be accessed locally. +on the Envoy deployment that corresponds to a Gateway so that it can be accessed locally. -`kubectl port-forward deploy/envoy-default-eg -n envoy-gateway-system 19000:19000` +`kubectl port-forward deploy/envoy-${GATEWAY_NAMESPACE}-${GATEWAY_NAME} -n envoy-gateway-system 19000:19000` Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. From 5105ab3fbd4b0d5736284a8b9987fdc520b85afa Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 10:13:52 -0700 Subject: [PATCH 031/113] Fixes Calculating ParentRef Status Conditions (#563) * Fixes Calculating ParentRef Status Conditions Signed-off-by: danehans * Adds condition len check to TLSRoute Signed-off-by: danehans Signed-off-by: danehans --- ...-with-header-filter-empty-headers.out.yaml | 6 +- ...ith-header-filter-invalid-headers.out.yaml | 6 +- ...th-header-filter-no-valid-headers.out.yaml | 6 +- ...edirect-filter-invalid-filter-type.in.yaml | 9 +- ...direct-filter-invalid-filter-type.out.yaml | 17 +-- ...th-redirect-filter-invalid-scheme.out.yaml | 6 +- ...th-redirect-filter-invalid-status.out.yaml | 6 +- ...rewrite-filter-invalid-filter-type.in.yaml | 41 +++++++ ...ewrite-filter-invalid-filter-type.out.yaml | 100 ++++++++++++++++++ internal/gatewayapi/translator.go | 27 ++++- 10 files changed, 194 insertions(+), 30 deletions(-) create mode 100644 internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml index ae0cd811cad..cdf6c1f2f49 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml @@ -66,9 +66,9 @@ httpRoutes: controllerName: gateway.envoyproxy.io/gatewayclass-controller conditions: - type: Accepted - status: "True" - reason: Accepted - message: Route is accepted + status: "False" + reason: UnsupportedValue + message: RequestHeaderModifier Filter cannot set a header with an empty name xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml index 29a05459042..2aec958edfb 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml @@ -66,9 +66,9 @@ httpRoutes: controllerName: gateway.envoyproxy.io/gatewayclass-controller conditions: - type: Accepted - status: "True" - reason: Accepted - message: Route is accepted + status: "False" + reason: UnsupportedValue + message: "RequestHeaderModifier Filter cannot set headers with a '/' or ':' character in them. Header: 'example:1'" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml index 3892e0bc6c9..399ca601319 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml @@ -61,9 +61,9 @@ httpRoutes: controllerName: gateway.envoyproxy.io/gatewayclass-controller conditions: - type: Accepted - status: "True" - reason: Accepted - message: Route is accepted + status: "False" + reason: UnsupportedValue + message: "RequestHeaderModifier Filter did not provide valid configuration to add/set/remove any headers" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.in.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.in.yaml index 32d8717c5fc..e40076f5b56 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.in.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.in.yaml @@ -35,8 +35,9 @@ httpRoutes: - name: service-1 port: 8080 filters: - - type: UnsupportedType - requestRedirect: - scheme: https - statusCode: 301 + - type: ExtensionRef + extensionRef: + group: unsupported.group.io + kind: UnsupportedKind + name: unsupported diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml index 3127021f040..0217bbf5716 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml @@ -47,10 +47,11 @@ httpRoutes: - name: service-1 port: 8080 filters: - - type: UnsupportedType - requestRedirect: - scheme: https - statusCode: 301 + - type: ExtensionRef + extensionRef: + group: unsupported.group.io + kind: UnsupportedKind + name: unsupported status: parents: - parentRef: @@ -60,9 +61,9 @@ httpRoutes: controllerName: gateway.envoyproxy.io/gatewayclass-controller conditions: - type: Accepted - status: "True" - reason: Accepted - message: Route is accepted + status: "False" + reason: UnsupportedValue + message: "Unknown custom filter type: ExtensionRef" xdsIR: envoy-gateway-gateway-1: http: @@ -81,7 +82,7 @@ xdsIR: # I believe the correct way to handle an invalid filter should be to allow the HTTPRoute to function # normally but leave out the filter config and set the status, but this behaviour can be changed. directResponse: - body: "Unknown custom filter type: UnsupportedType" + body: "Unknown custom filter type: ExtensionRef" statusCode: 500 infraIR: envoy-gateway-gateway-1: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml index 630f9fb75ec..c211bfb8e69 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml @@ -60,9 +60,9 @@ httpRoutes: controllerName: gateway.envoyproxy.io/gatewayclass-controller conditions: - type: Accepted - status: "True" - reason: Accepted - message: Route is accepted + status: "False" + reason: UnsupportedValue + message: "Scheme: unknown is unsupported, only 'https' and 'http' are supported" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml index dbb2df4af0c..007254e44ab 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml @@ -60,9 +60,9 @@ httpRoutes: controllerName: gateway.envoyproxy.io/gatewayclass-controller conditions: - type: Accepted - status: "True" - reason: Accepted - message: Route is accepted + status: "False" + reason: UnsupportedValue + message: "Status code 666 is invalid, only 302 and 301 are supported" xdsIR: envoy-gateway-gateway-1: http: diff --git a/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.in.yaml b/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.in.yaml new file mode 100644 index 00000000000..17a0293da6b --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.in.yaml @@ -0,0 +1,41 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + hostname: "*.envoyproxy.io" + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 + filters: + - type: URLRewrite + urlRewrite: + hostname: urlrewrite.envoyproxy.io + diff --git a/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml b/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml new file mode 100644 index 00000000000..8ff359b4867 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml @@ -0,0 +1,100 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + hostname: "*.envoyproxy.io" + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 + filters: + - type: URLRewrite + urlRewrite: + hostname: urlrewrite.envoyproxy.io + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + sectionName: http + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "False" + reason: UnsupportedValue + message: "Unsupported filter type: URLRewrite" +xdsIR: + envoy-gateway-gateway-1: + http: + - name: envoy-gateway-gateway-1-http + address: 0.0.0.0 + port: 10080 + hostnames: + - "*.envoyproxy.io" + routes: + - name: default-httproute-1-rule-0-match-0-gateway.envoyproxy.io + pathMatch: + prefix: "/" + headerMatches: + - name: ":authority" + exact: gateway.envoyproxy.io + # I believe the correct way to handle an invalid filter should be to allow the HTTPRoute to function + # normally but leave out the filter config and set the status, but this behaviour can be changed. + directResponse: + body: "Unsupported filter type: URLRewrite" + statusCode: 500 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:v1.23-latest + listeners: + - address: "" + ports: + - name: http + protocol: "HTTP" + containerPort: 10080 + servicePort: 80 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 80ed034e8ba..6212fd9154e 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -1025,7 +1025,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways "RequestHeaderModifier Filter did not provide valid configuration to add/set/remove any headers", ) } - default: + case v1beta1.HTTPRouteFilterExtensionRef: // "If a reference to a custom filter type cannot be resolved, the filter MUST NOT be skipped. // Instead, requests that would have been processed by that filter MUST receive a HTTP error response." errMsg := fmt.Sprintf("Unknown custom filter type: %s", filter.Type) @@ -1039,6 +1039,19 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways Body: &errMsg, StatusCode: 500, } + default: + // Unsupported filters. + errMsg := fmt.Sprintf("Unsupported filter type: %s", filter.Type) + parentRef.SetCondition(httpRoute, + v1beta1.RouteConditionAccepted, + metav1.ConditionFalse, + v1beta1.RouteReasonUnsupportedValue, + errMsg, + ) + directResponse = &ir.DirectResponse{ + Body: &errMsg, + StatusCode: 500, + } } } @@ -1181,7 +1194,11 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways v1beta1.RouteReasonNoMatchingListenerHostname, "There were no hostname intersections between the HTTPRoute and this parent ref's Listener(s).", ) - } else { + } + + // If no negative conditions have been set, the route is considered "Accepted=True". + if parentRef.httpRoute != nil && + len(parentRef.httpRoute.Status.Parents[parentRef.routeParentStatusIdx].Conditions) == 0 { parentRef.SetCondition(httpRoute, v1beta1.RouteConditionAccepted, metav1.ConditionTrue, @@ -1361,7 +1378,11 @@ func (t *Translator) ProcessTLSRoutes(tlsRoutes []*v1alpha2.TLSRoute, gateways [ v1beta1.RouteReasonNoMatchingListenerHostname, "There were no hostname intersections between the HTTPRoute and this parent ref's Listener(s).", ) - } else { + } + + // If no negative conditions have been set, the route is considered "Accepted=True". + if parentRef.tlsRoute != nil && + len(parentRef.tlsRoute.Status.Parents[parentRef.routeParentStatusIdx].Conditions) == 0 { parentRef.SetCondition(tlsRoute, v1beta1.RouteConditionAccepted, metav1.ConditionTrue, From 29d5189db6fa18e1a30e3f9085d1405feb34add8 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 18 Oct 2022 02:22:30 +0800 Subject: [PATCH 032/113] chore: add markdown linters (#572) Signed-off-by: bitliu Signed-off-by: bitliu --- .github/markdown_lint_config.json | 52 +++++++++++++++++++++++++++++++ .github/workflows/docs.yaml | 28 +++++++++++++---- docs/dev/RELEASE.md | 28 ++++++++++++++--- docs/user/HTTP_ROUTING.md | 23 ++++++++++++-- 4 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 .github/markdown_lint_config.json diff --git a/.github/markdown_lint_config.json b/.github/markdown_lint_config.json new file mode 100644 index 00000000000..4550512017d --- /dev/null +++ b/.github/markdown_lint_config.json @@ -0,0 +1,52 @@ +{ + "MD001": true, + "MD002": false, + "MD003": false, + "MD004": false, + "MD005": false, + "MD006": false, + "MD007": false, + "MD008": false, + "MD009": false, + "MD010": false, + "MD011": false, + "MD012": false, + "MD013": false, + "MD014": false, + "MD015": false, + "MD016": false, + "MD017": false, + "MD018": false, + "MD019": false, + "MD020": false, + "MD021": false, + "MD022": false, + "MD023": false, + "MD024": false, + "MD025": false, + "MD026": false, + "MD027": false, + "MD028": false, + "MD029": false, + "MD030": false, + "MD031": true, + "MD032": false, + "MD033": false, + "MD034": false, + "MD035": false, + "MD036": false, + "MD037": true, + "MD038": true, + "MD039": false, + "MD040": false, + "MD041": false, + "MD042": false, + "MD043": false, + "MD044": false, + "MD045": false, + "MD046": false, + "MD047": false, + "MD048": false, + "MD049": false, + "MD050": false +} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f8ba90a3e68..e070a532ffb 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -6,15 +6,30 @@ on: - "release-v*" paths-ignore: - "**/*.png" - # pull_request: - # branches: - # - "main" - # - "release-v*" - # paths-ignore: - # - "**/*.png" + pull_request: + branches: + - "main" + - "release-v*" + paths-ignore: + - "**/*.png" + jobs: + docs-lint: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Run markdown linter + uses: nosborn/github-action-markdown-cli@v3.1.0 + with: + files: docs/* + config_file: ".github/markdown_lint_config.json" + docs-build: + if: github.event_name == 'push' runs-on: ubuntu-latest + needs: docs-lint steps: - uses: actions/checkout@v3 - uses: ./tools/github-actions/setup-deps @@ -33,6 +48,7 @@ jobs: # This workflow contains a single job called "build" docs-publish: + if: github.event_name == 'push' runs-on: ubuntu-latest needs: docs-build diff --git a/docs/dev/RELEASE.md b/docs/dev/RELEASE.md index 674e5c35e6c..a94e384495c 100644 --- a/docs/dev/RELEASE.md +++ b/docs/dev/RELEASE.md @@ -1,7 +1,9 @@ -## Introduction +# Release Process + This document guides maintainers through the process of creating an Envoy Gateway release. ## Prerequisites + - Permissions to push to the Envoy Gateway repository. ## Creating a Minor Release @@ -13,9 +15,11 @@ This document guides maintainers through the process of creating an Envoy Gatewa before cutting the release. 4. Create a new release branch from `main`. The release branch should be named `release/v${MAJOR_VERSION}.${MINOR_VERSION}.0`, e.g. `release/v0.3.0`. + ```shell git checkout -b release/v0.3.0 ``` + 5. Push the branch to the Envoy Gateway repo. 6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. See [PR 481][] as a reference for the required changes. @@ -23,21 +27,25 @@ This document guides maintainers through the process of creating an Envoy Gatewa Do not proceed until your PR is merged. 8. Confirm that the [release workflow][] for your PR completed successfully. 9. Tag the head of your release branch with the release tag. For example: + ```shell git tag -a v0.3.0 -m 'Envoy Gateway v0.3.0 Release' ``` + 10. Push the tag to the Envoy Gateway repository. + ```shell git push --tags ``` + 11. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. 12. Confirm that the [release workflow][] completed successfully. 13. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. 14. Confirm that the [release][] was created. 15. Confirm that the steps in the [Quickstart Guide][] work as expected. 16. [Generate][] the GitHub changelog. -18. Submit a PR to merge the Quickstart Guide changes from the release branch into the main branch. -19. If you find any bugs in this process, please create an issue. +17. Submit a PR to merge the Quickstart Guide changes from the release branch into the main branch. +18. If you find any bugs in this process, please create an issue. ## Creating a Release Candidate @@ -48,13 +56,17 @@ This document guides maintainers through the process of creating an Envoy Gatewa before cutting the release candidate. 4. Tag the head of the main branch with the release candidate number. The tag should be named `v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER}`. For example: + ```shell git tag -a v0.3.0-rc.1 -m 'Envoy Gateway v0.3.0-rc.1 Release Candidate' ``` + 5. Push the tag to the Envoy Gateway repository. + ```shell git push --tags ``` + 6. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. 7. Confirm that the [release workflow][] completed successfully. 8. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. @@ -62,20 +74,26 @@ This document guides maintainers through the process of creating an Envoy Gatewa 10. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test the quickstart steps using the release candidate by manually updating the links. 11. [Generate][] the GitHub changelog. -13. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. -14. If you find any bugs in this process, please create an issue. +12. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. +13. If you find any bugs in this process, please create an issue. ## Announcing the Release + It's important that the world knows about the release. Follow the steps to announce the release. + 1. Set the release information in the Envoy Gateway Slack channel. For example: + ```shell Envoy Gateway v0.3.0 has been released: https://github.com/envoyproxy/gateway/releases/tag/v0.3.0 ``` + 2. Send a message to the Envoy Gateway Slack channel. For example: + ```shell I am pleased to announce the release of Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0. The release would not be possible without all the support from the Envoy Gateway community... ``` + Include a sentence or two that highlights key aspects of the release. [release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes diff --git a/docs/user/HTTP_ROUTING.md b/docs/user/HTTP_ROUTING.md index 004122bcb4b..3d19062abfd 100644 --- a/docs/user/HTTP_ROUTING.md +++ b/docs/user/HTTP_ROUTING.md @@ -7,9 +7,11 @@ Services. To learn more about HTTP routing, refer to the [Gateway API documentat Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and then install the example resources used for this guide. + ```shell kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0/examples/kubernetes/http-routing.yaml ``` + The manifest installs a [GatewayClass][], [Gateway][], four Deployments, four Services, and three HTTPRoute resources. The GatewayClass is a cluster-scoped resource that represents a class of Gateways that can be instantiated. Envoy @@ -17,42 +19,51 @@ Gateway is configured by default to manage GatewayClasses with `controllerName: gateway.envoyproxy.io/gatewayclass-controller`. Check the status of the GatewayClass: + ```shell kubectl get gc --selector=example=http-routing ``` + The status should reflect "Accepted=True", indicating Envoy Gateway is managing the GatewayClass. A Gateway represents configuration of infrastructure. When a Gateway is created, [Envoy proxy][] infrastructure is provisioned or configured by Envoy Gateway. The `gatewayClassName` defines the name of a GatewayClass used by this Gateway. Check the status of the Gateway: + ```shell kubectl get gateways --selector=example=http-routing ``` + The status should reflect "Ready=True", indicating the Envoy proxy infrastructure has been provisioned. The status also provides the address of the Gateway. This address is used later in the guide to test connectivity to proxied backend services. -The three HTTPRoute resources create routing rules on the Gateway. In order to receive traffic from a Gateway, +The three HTTPRoute resources create routing rules on the Gateway. In order to receive traffic from a Gateway, an HTTPRoute must be configured with `parentRefs` which reference the parent Gateway(s) that it should be attached to. An HTTPRoute can match against a [single set of hostnames][spec]. These hostnames are matched before any other matching within the HTTPRoute takes place. Since `example.com`, `foo.example.com`, and `bar.example.com` are separate hosts with different routing requirements, each is deployed as its own HTTPRoute - `example-route, ``foo-route`, and `bar-route`. Check the status of the HTTPRoutes: + ```shell kubectl get httproutes --selector=example=http-routing -o yaml ``` + The status for each HTTPRoute should surface "Accepted=True" and a `parentRef` that references the example Gateway. The `example-route` matches any traffic for "example.com" and forwards it to the "example-svc" Service. Before testing HTTP routing to the `example-svc` backend, get the Gateway's address. + ```shell export GATEWAY_HOST=$(kubectl get gateway/example-gateway -o jsonpath='{.status.addresses[0].value}') ``` Test HTTP routing to the `example-svc` backend. + ```shell curl -vvv --header "Host: example.com" "http://${GATEWAY_HOST}/" ``` + A `200` status code should be returned and the body should include `"pod": "example-backend-*"` indicating the traffic was routed to the example backend service. If you change the hostname to a hostname not represented in any of the HTTPRoutes, e.g. "www.example.com", the HTTP traffic will not be routed and a `404` should be returned. @@ -60,32 +71,39 @@ HTTPRoutes, e.g. "www.example.com", the HTTP traffic will not be routed and a `4 The `foo-route` matches any traffic for `foo.example.com` and applies its routing rules to forward the traffic to the "foo-svc" Service. Since there is only one path prefix match for `/login`, only `foo.example.com/login/*` traffic will be forwarded. Test HTTP routing to the `foo-svc` backend. + ```shell curl -vvv --header "Host: foo.example.com" "http://${GATEWAY_HOST}/login" ``` A `200` status code should be returned and the body should include `"pod": "foo-backend-*"` indicating the traffic was routed to the foo backend service. Traffic to any other paths that do not begin with `/login` will not be matched by -this HTTPRoute. Test this by removing `/login` from the request. +this HTTPRoute. Test this by removing `/login` from the request. + ```shell curl -vvv --header "Host: foo.example.com" "http://${GATEWAY_HOST}/" ``` + The HTTP traffic will not be routed and a `404` should be returned. Similarly, the `bar-route` HTTPRoute matches traffic for `bar.example.com`. All traffic for this hostname will be evaluated against the routing rules. The most specific match will take precedence which means that any traffic with the `env:canary` header will be forwarded to `bar-svc-canary` and if the header is missing or not `canary` then it'll be forwarded to `bar-svc`. Test HTTP routing to the `bar-svc` backend. + ```shell curl -vvv --header "Host: bar.example.com" "http://${GATEWAY_HOST}/" ``` + A `200` status code should be returned and the body should include `"pod": "bar-backend-*"` indicating the traffic was routed to the foo backend service. Test HTTP routing to the `bar-canary-svc` backend by adding the `env: canary` header to the request. + ```shell curl -vvv --header "Host: bar.example.com" --header "env: canary" "http://${GATEWAY_HOST}/" ``` + A `200` status code should be returned and the body should include `"pod": "bar-canary-backend-*"` indicating the traffic was routed to the foo backend service. @@ -95,4 +113,3 @@ traffic was routed to the foo backend service. [Gateway]: https://gateway-api.sigs.k8s.io/api-types/gateway/ [Envoy proxy]: https://www.envoyproxy.io/ [spec]: /references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteSpec -[svc]:https://kubernetes.io/docs/concepts/services-networking/service/ From 9f5a4721e6258a540125429b4d25c2afcc565986 Mon Sep 17 00:00:00 2001 From: Amila Senadheera Date: Tue, 18 Oct 2022 01:57:28 +0530 Subject: [PATCH 033/113] Fix Secrets deletion logic in gateway reconcile method (#579) * fix delete secret condition issue in gateway reconcile method Signed-off-by: Amila Senadheera * matched secrets assigned to outer loop variable Signed-off-by: Amila Senadheera * Revert: matched secrets assigned to outer loop variable Signed-off-by: Amila Senadheera * Reverting to the state of the branch at 27ec819c7f5 Signed-off-by: Amila Senadheera Signed-off-by: Amila Senadheera --- internal/provider/kubernetes/gateway.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index e740202bdfb..6e19957b6e9 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -385,13 +385,13 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req switch { case err != nil: r.log.Error(err, "failed to verify if other gateways reference secret") - case referenced: + case !referenced: r.log.Info("no other gateways reference secret; deleting from resource map", "namespace", secret.Namespace, "name", secret.Name) key := utils.NamespacedName(&secret) r.resources.Secrets.Delete(key) default: - r.log.Info("no other gateways reference secret; deleting from resource map", + r.log.Info("other gateways reference secret; keeping the secret in the resource map", "namespace", secret.Namespace, "name", secret.Name) } } From 71b23eaecefaa3ee741c0ba520e73875de2976e3 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 13:27:50 -0700 Subject: [PATCH 034/113] Updates Spellcheck Ignore (#584) * Updates Spellcheck Ignore Signed-off-by: danehans * Resolved @Arko Feedback Signed-off-by: danehans Signed-off-by: danehans --- tools/linter/codespell/.codespell.skip | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/linter/codespell/.codespell.skip b/tools/linter/codespell/.codespell.skip index 0f7ff043e14..319d3a78dfe 100644 --- a/tools/linter/codespell/.codespell.skip +++ b/tools/linter/codespell/.codespell.skip @@ -8,6 +8,7 @@ *.jpg *.ico *.svg +./docs/html/* go.mod go.sum bin From abc086a8e35fa6083c056d6cc20f9c455a25bfa8 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 14:30:52 -0700 Subject: [PATCH 035/113] Updates Release Branch Naming (#582) * Updates Release Branch Naming Signed-off-by: danehans * Renames file Signed-off-by: danehans Signed-off-by: danehans --- docs/dev/{RELEASE.md => releasing.md} | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename docs/dev/{RELEASE.md => releasing.md} (94%) diff --git a/docs/dev/RELEASE.md b/docs/dev/releasing.md similarity index 94% rename from docs/dev/RELEASE.md rename to docs/dev/releasing.md index a94e384495c..5acb2c210a9 100644 --- a/docs/dev/RELEASE.md +++ b/docs/dev/releasing.md @@ -14,10 +14,10 @@ This document guides maintainers through the process of creating an Envoy Gatewa 3. Submit a [Pull Request][] to merge the release notes into the main branch. This should be the last commit to main before cutting the release. 4. Create a new release branch from `main`. The release branch should be named - `release/v${MAJOR_VERSION}.${MINOR_VERSION}.0`, e.g. `release/v0.3.0`. + `release/v${MAJOR_VERSION}.${MINOR_VERSION}`, e.g. `release/v0.3`. ```shell - git checkout -b release/v0.3.0 + git checkout -b release/v0.3 ``` 5. Push the branch to the Envoy Gateway repo. @@ -32,6 +32,8 @@ This document guides maintainers through the process of creating an Envoy Gatewa git tag -a v0.3.0 -m 'Envoy Gateway v0.3.0 Release' ``` + __Note:__ The tag version differs from the release branch by including the `.0` patch version. + 10. Push the tag to the Envoy Gateway repository. ```shell @@ -79,7 +81,7 @@ This document guides maintainers through the process of creating an Envoy Gatewa ## Announcing the Release -It's important that the world knows about the release. Follow the steps to announce the release. +It's important that the world knows about the release. Use the following steps to announce the release. 1. Set the release information in the Envoy Gateway Slack channel. For example: From fbb2efddb857fbcda1d7bf4bb668ebadabe494e5 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 18 Oct 2022 05:31:44 +0800 Subject: [PATCH 036/113] chore: use conformance echo-server (#578) Signed-off-by: bitliu Signed-off-by: bitliu --- examples/kubernetes/quickstart.yaml | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/kubernetes/quickstart.yaml b/examples/kubernetes/quickstart.yaml index b045025c85a..59b1cc7d49d 100644 --- a/examples/kubernetes/quickstart.yaml +++ b/examples/kubernetes/quickstart.yaml @@ -19,51 +19,51 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: httpbin + name: backend --- apiVersion: v1 kind: Service metadata: - name: httpbin + name: backend labels: - app: httpbin - service: httpbin + app: backend + service: backend spec: ports: - name: http - port: 80 - targetPort: 80 + port: 3000 + targetPort: 3000 selector: - app: httpbin + app: backend --- apiVersion: apps/v1 kind: Deployment metadata: - name: httpbin + name: backend spec: replicas: 1 selector: matchLabels: - app: httpbin + app: backend version: v1 template: metadata: labels: - app: httpbin + app: backend version: v1 spec: - serviceAccountName: httpbin + serviceAccountName: backend containers: - - image: docker.io/kennethreitz/httpbin + - image: gcr.io/k8s-staging-ingressconformance/echoserver:v20220815-e21d1a4 imagePullPolicy: IfNotPresent - name: httpbin + name: backend ports: - - containerPort: 80 + - containerPort: 3000 --- apiVersion: gateway.networking.k8s.io/v1beta1 kind: HTTPRoute metadata: - name: httpbin + name: backend spec: parentRefs: - name: eg @@ -73,8 +73,8 @@ spec: - backendRefs: - group: "" kind: Service - name: httpbin - port: 80 + name: backend + port: 3000 weight: 1 matches: - path: From 5f118ec931874196eb2b44f0a8af57369951139d Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 17 Oct 2022 14:46:23 -0700 Subject: [PATCH 037/113] Reorganizes Dev Guide and Adds Conformance for Mac (#489) Signed-off-by: danehans --- DEVELOPER.md | 87 --------------------------- README.md | 2 +- docs/dev/README.md | 129 ++++++++++++++++++++++++++++++++++++++++ docs/user/QUICKSTART.md | 2 +- 4 files changed, 131 insertions(+), 89 deletions(-) delete mode 100644 DEVELOPER.md create mode 100644 docs/dev/README.md diff --git a/DEVELOPER.md b/DEVELOPER.md deleted file mode 100644 index c669eeedce1..00000000000 --- a/DEVELOPER.md +++ /dev/null @@ -1,87 +0,0 @@ -# Developer documentation - -Envoy Gateway is built using a [make][make]-based build system. Our CI is based on [Github Actions][gha] -(see: [workflows](.github/workflows)). - -## Prerequisites - -### go -* Version: 1.18.2 -* Installation Guide: https://go.dev/doc/install - -### make -* Recommended Version: 4.0 or later -* Installation Guide: https://www.gnu.org/software/make - -### docker -* Optional when you want to build a Docker image or run `make` inside Docker. -* Recommended Version: 20.10.16 -* Installation Guide: https://docs.docker.com/engine/install - -### python3 -* Need a `python3` program -* Must have a functioning `venv` module; this is part of the standard - library, but some distributions (such as Debian and Ubuntu) replace - it with a stub and require you to install a `python3-venv` package - separately. - -## Quick start -* Run `make help` to see all the available targets to build, test and run `envoy-gateway`. - -### Building the `envoy-gateway` binary -* Run `make build` to build the binary that gets generated in the `bin/` directory - -### Running tests -* Run `make test` to run the golang tests. - -### Running code linters -* Run `make lint` to make sure your code passes all the linter checks. - -### Building and Pushing the Image -* Run `IMAGE=docker.io/you/gateway-dev make image` to build the docker image. -* Run `IMAGE=docker.io/you/gateway-dev make push-multiarch` to build and push the multi-arch docker image. - -**_NOTE:_** Replace `IMAGE` with your registry's image name. - -### Creating a Kind Cluster to deploy Envoy Gateway -* Run `make create-cluster` to create a [Kind][kind] cluster. -* Run `make kube-install-image` to build an image and load it into the Kind cluster. - -**_NOTE:_** Envoy Gateway is tested against Kubernetes v1.24.0. - -### Deploying Envoy Gateway in Kubernetes -* Run `IMAGE=envoyproxy/gateway-dev TAG=latest make kube-deploy` to deploy Envoy Gateway resources, including the Gateway API CRDs, -with the `envoyproxy/gateway-dev:latest` Envoy Gateway image into a Kubernetes cluster (linked to the current kube context). -* Run `make kube-undeploy` to delete the resources from the cluster created using `kube-deploy`. - -**_NOTE:_** Replace `IMAGE` with your registry's image name. - -### Configure a demo setup -* Run `make kube-demo` to deploy a demo backend service, gatewayclass, gateway and httproute resource -(similar to steps outlined in the [Quickstart](https://github.com/envoyproxy/gateway/blob/main/docs/user/QUICKSTART.md) docs) and test the configuration. -* Run `make kube-demo-undeploy` to delete the resources created by the `make kube-demo` command. - -### Run Gateway API Conformance Tests -* Run `make conformance` to run Gateway API Conformance tests using `envoy-gateway` in a -local Kind cluster. Go [here](https://gateway-api.sigs.k8s.io/concepts/conformance/) to learn -more about the tests. - -**_NOTE:_** Conformance tests against a kind cluster is currently unsupported on Mac computers. -As a workaround, you could run this against your own Kubernetes cluster (such as Kubernetes on Docker Desktop) using this command - -`IMAGE=docker.io/you/gateway-dev make push-multiarch && IMAGE=docker.io/you/gateway-dev make kube-deploy && make run-conformance` -which builds and pushes the Envoy-Gateway image to your hub, deploys Envoy Gateway resources into your cluster -and runs the Gateway API conformance tests. - -### Debugging the Envoy Config -An easy way to view the envoy config that Envoy Gateway is using is to port-forward to the admin interface port (currently `19000`) -on the Envoy deployment that corresponds to a Gateway so that it can be accessed locally. - -`kubectl port-forward deploy/envoy-${GATEWAY_NAMESPACE}-${GATEWAY_NAME} -n envoy-gateway-system 19000:19000` - -Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. - -There are many other endpoints on the [Envoy admin interface](https://www.envoyproxy.io/docs/envoy/v1.23.0/operations/admin#operations-admin-interface) that may be helpful when debugging. - -[make]: https://www.gnu.org/software/make/ -[gha]: https://docs.github.com/en/actions -[kind]: https://kind.sigs.k8s.io/ diff --git a/README.md b/README.md index cc9ecb13788..a2d293ab0f5 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Kubernetes-based application gateway. * [Code of conduct](CODE_OF_CONDUCT.md) * [Contributing guide](CONTRIBUTING.md) -* [Developer guide](DEVELOPER.md) +* [Developer guide](docs/dev/README.md) ## Community Meeting diff --git a/docs/dev/README.md b/docs/dev/README.md new file mode 100644 index 00000000000..8cc7c23f281 --- /dev/null +++ b/docs/dev/README.md @@ -0,0 +1,129 @@ +# Developer Guide + +Envoy Gateway is built using a [make][]-based build system. Our CI is based on [Github Actions][] using [workflows][]. + +## Prerequisites + +### go + +* Version: 1.18.2 +* Installation Guide: https://go.dev/doc/install + +### make + +* Recommended Version: 4.0 or later +* Installation Guide: https://www.gnu.org/software/make + +### docker + +* Optional when you want to build a Docker image or run `make` inside Docker. +* Recommended Version: 20.10.16 +* Installation Guide: https://docs.docker.com/engine/install + +### python3 + +* Need a `python3` program +* Must have a functioning `venv` module; this is part of the standard + library, but some distributions (such as Debian and Ubuntu) replace + it with a stub and require you to install a `python3-venv` package + separately. + +## Quickstart + +* Run `make help` to see all the available targets to build, test and run Envoy Gateway. + +### Building + +* Run `make build` to build the Envoy Gateway binary. __Note:__ The binary gets generated in the `bin/` directory + +### Testing + +* Run `make test` to run the golang tests. + +### Running Linters + +* Run `make lint` to make sure your code passes all the linter checks. + +### Building and Pushing the Image + +* Run `IMAGE=docker.io/you/gateway-dev make image` to build the docker image. +* Run `IMAGE=docker.io/you/gateway-dev make push-multiarch` to build and push the multi-arch docker image. + +__Note:__ Replace `IMAGE` with your registry's image name. + +### Deploying Envoy Gateway for Test/Dev + +* Run `make create-cluster` to create a [Kind][] cluster. + +#### Option 1: Use the Latest [gateway-dev][] Image + +* Run `TAG=latest make kube-deploy` to deploy Envoy Gateway in the Kind cluster using the latest image. Replace `latest` + to use a different image tag. + +#### Option 2: Use a Custom Image + +* Run `make kube-install-image` to build an image from the tip of your current branch and load it in the Kind cluster. +* Run `make kube-deploy` to install Envoy Gateway into the Kind cluster using your custom image. + +### Deploying Envoy Gateway in Kubernetes + +* Run `TAG=latest make kube-deploy` to deploy Envoy Gateway using the latest image into a Kubernetes cluster (linked to + the current kube context). Preface the command with `IMAGE` or replace `TAG` to use a different Envoy Gateway image or + tag. +* Run `make kube-undeploy` to uninstall Envoy Gateway from the cluster. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. + +### Demo Setup + +* Run `make kube-demo` to deploy a demo backend service, gatewayclass, gateway and httproute resource +(similar to steps outlined in the [Quickstart][] docs) and test the configuration. +* Run `make kube-demo-undeploy` to delete the resources created by the `make kube-demo` command. + +### Run Gateway API Conformance Tests + +The commands below deploy Envoy Gateway to a Kubernetes cluster and run the Gateway API conformance tests. Refer to the +Gateway API [conformance homepage][] to learn more about the tests. If Envoy Gateway is already installed, run +`TAG=latest make run-conformance` to run the conformance tests. + +#### On a Linux Host + +* Run `TAG=latest make conformance` to create a Kind cluster, install Envoy Gateway using the latest [gateway-dev][] + image, and run Gateway API conformance tests. + +#### On a Mac Host + +Since Mac doesn't support [directly exposing][] the Docker network to the Mac host, use one of the following +workarounds to run conformance tests: + +* Deploy your own Kubernetes cluster or use Docker Desktop with [Kubernetes support][] and then run + `TAG=latest make kube-deploy run-conformance`. This will install Envoy Gateway using the latest [gateway-dev][] image + to the Kubernetes cluster using the current kubectl context and run the conformance tests. Use `make kube-undeploy` to + uninstall Envoy Gateway. +* Install and run [Docker Mac Net Connect][mac_connect] and then run `TAG=latest make conformance`. + +__Note:__ Preface commands with `IMAGE` or replace `TAG` to use a different Envoy Gateway image or tag. If `TAG` +is unspecified, the short SHA of your current branch is used. + +### Debugging the Envoy Config + +An easy way to view the envoy config that Envoy Gateway is using is to port-forward to the admin interface port (currently `19000`) +on the Envoy deployment that corresponds to a Gateway so that it can be accessed locally. + +`kubectl port-forward deploy/envoy-${GATEWAY_NAMESPACE}-${GATEWAY_NAME} -n envoy-gateway-system 19000:19000` + +Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. + +There are many other endpoints on the [Envoy admin interface][] that may be helpful when debugging. + +[Quickstart]: https://github.com/envoyproxy/gateway/blob/main/docs/user/QUICKSTART.md +[make]: https://www.gnu.org/software/make/ +[Github Actions]: https://docs.github.com/en/actions +[workflows]: .github/workflows +[Kind]: https://kind.sigs.k8s.io/ +[conformance homepage]: https://gateway-api.sigs.k8s.io/concepts/conformance/ +[directly exposing]: https://kind.sigs.k8s.io/docs/user/loadbalancer/ +[Kubernetes support]: https://docs.docker.com/desktop/kubernetes/ +[gateway-dev]: https://hub.docker.com/r/envoyproxy/gateway-dev/tags +[mac_connect]: https://github.com/chipmk/docker-mac-net-connect +[Envoy admin interface]: https://www.envoyproxy.io/docs/envoy/v1.23.0/operations/admin#operations-admin-interface diff --git a/docs/user/QUICKSTART.md b/docs/user/QUICKSTART.md index 931ca88bd6c..c826f410e0b 100644 --- a/docs/user/QUICKSTART.md +++ b/docs/user/QUICKSTART.md @@ -124,6 +124,6 @@ kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0 ## Next Steps -Checkout the [Developer Guide](../../DEVELOPER.md) to get involved in the project. +Checkout the [Developer Guide](../dev/README.md) to get involved in the project. [httpbin_methods]: https://httpbin.org/#/HTTP_Methods From b05a5267e6c15d35faa2e750c7b5fdef8521fd86 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 18 Oct 2022 23:42:45 +0800 Subject: [PATCH 038/113] fix: add pod / ns env to echo server (#592) Signed-off-by: bitliu Signed-off-by: bitliu --- examples/kubernetes/quickstart.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/kubernetes/quickstart.yaml b/examples/kubernetes/quickstart.yaml index 59b1cc7d49d..21ab775b04b 100644 --- a/examples/kubernetes/quickstart.yaml +++ b/examples/kubernetes/quickstart.yaml @@ -59,6 +59,15 @@ spec: name: backend ports: - containerPort: 3000 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace --- apiVersion: gateway.networking.k8s.io/v1beta1 kind: HTTPRoute From f919087a1ad08de06768aa1ea8a5c3ddd800e2f1 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 18 Oct 2022 23:49:31 +0800 Subject: [PATCH 039/113] feat: support latest release of EG (#569) Signed-off-by: bitliu --- .github/workflows/build_and_test.yaml | 10 +++-- .github/workflows/pre_release.yaml | 53 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/pre_release.yaml diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index b43e355c566..df574694a14 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -30,12 +30,14 @@ jobs: build-and-test: runs-on: ubuntu-latest + needs: [lint, gen-check] steps: - uses: actions/checkout@v3 - uses: ./tools/github-actions/setup-deps # test - - run: make go.test.coverage + - name: Run Coverage Tests + run: make go.test.coverage - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 with: @@ -45,7 +47,8 @@ jobs: verbose: true # build - - run: make build-multiarch + - name: Build Multiarch EG Binaries + run: make build-multiarch PLATFORMS="linux_amd64 linux_arm64" # conformance - name: Run Conformance Tests @@ -59,7 +62,8 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - run: make image.multiarch.setup + - name: Setup Multiarch Environment + run: make image.multiarch.setup - name: Build and Push EG Commit Image if: github.event_name == 'push' diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml new file mode 100644 index 00000000000..9d6b055478a --- /dev/null +++ b/.github/workflows/pre_release.yaml @@ -0,0 +1,53 @@ +name: Canary Release + +on: + push: + branches: + - "main" + paths-ignore: + - "**/*.png" + +jobs: + canary-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Generate Release Manifests + run: make generate-manifests IMAGE=envoyproxy/gateway-dev TAG=latest OUTPUT_DIR=release-artifacts + + # Ignore the error from the first time canary release deletion. + # We do not have the release at first, after that, the error will not appear again. + - name: Delete Canary Release + continue-on-error: true + run: | + gh release delete latest --repo $GITHUB_REPOSITORY + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository_owner }}/${{ github.event.repository.name }} + + # Same as above, ignore the error from the first time canary tag deletion. + # We do not have the tag at first, after that, the error will not appear again. + - name: Delete Canary Tag + continue-on-error: true + run: + gh api --method DELETE /repos/$GITHUB_REPOSITORY/git/refs/tags/latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository_owner }}/${{ github.event.repository.name }} + + - name: Recreate Canary Release and Tag + uses: softprops/action-gh-release@v1 + with: + draft: false + prerelease: true + tag_name: latest + files: | + release-artifacts/install.yaml + release-artifacts/quickstart.yaml + body: | + This is a "Canary Release" of **Envoy Gateway**, which contains the most recent commits on our main branch. + + Canary is **not stable**. + + It is only intended for developers wishing to try out the latest features in Envoy Gateway, some of which may not be fully implemented. From 9cda522117e211a62d73ae7c04e917731f509030 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Tue, 18 Oct 2022 08:49:44 -0700 Subject: [PATCH 040/113] Merge ir listeners into single xds listener (#574) * Merge ir listeners into single xds listener * Reuse an xds listener if one already exists for the same addr:port * Add filter chain matches with a unique filter chain and route config for each of these ir listeners, except for http listeners which share the same DefaultFilterChain and the same route config * removed the xds resource naming for now since it wasnt adding much value Signed-off-by: Arko Dasgupta * add conformance tests Signed-off-by: Arko Dasgupta Signed-off-by: Arko Dasgupta --- internal/xds/translator/cluster.go | 2 +- internal/xds/translator/listener.go | 176 +++++++++++------- internal/xds/translator/route.go | 4 +- .../xds-ir/multiple-listeners-same-port.yaml | 66 +++++++ .../http-route-direct-response.clusters.yaml | 4 +- .../http-route-direct-response.listeners.yaml | 8 +- .../http-route-direct-response.routes.yaml | 4 +- .../xds-ir/http-route-redirect.clusters.yaml | 4 +- .../xds-ir/http-route-redirect.listeners.yaml | 8 +- .../xds-ir/http-route-redirect.routes.yaml | 4 +- .../http-route-request-headers.clusters.yaml | 4 +- .../http-route-request-headers.listeners.yaml | 8 +- .../http-route-request-headers.routes.yaml | 6 +- ...ute-weighted-invalid-backend.clusters.yaml | 4 +- ...te-weighted-invalid-backend.listeners.yaml | 8 +- ...route-weighted-invalid-backend.routes.yaml | 6 +- .../out/xds-ir/http-route.clusters.yaml | 4 +- .../out/xds-ir/http-route.listeners.yaml | 8 +- .../out/xds-ir/http-route.routes.yaml | 6 +- ...multiple-listeners-same-port.clusters.yaml | 102 ++++++++++ ...ultiple-listeners-same-port.listeners.yaml | 127 +++++++++++++ .../multiple-listeners-same-port.routes.yaml | 38 ++++ .../multiple-listeners-same-port.secrets.yaml | 12 ++ .../out/xds-ir/simple-tls.clusters.yaml | 4 +- .../out/xds-ir/simple-tls.listeners.yaml | 17 +- .../out/xds-ir/simple-tls.routes.yaml | 6 +- .../out/xds-ir/simple-tls.secrets.yaml | 2 +- .../tls-route-passthrough.clusters.yaml | 4 +- .../tls-route-passthrough.listeners.yaml | 4 +- internal/xds/translator/translator.go | 110 +++++++---- internal/xds/translator/translator_test.go | 4 + test/conformance/conformance_test.go | 2 + 32 files changed, 607 insertions(+), 159 deletions(-) create mode 100644 internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index b98d5dfe806..e63d283eb9d 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -23,7 +23,7 @@ func buildXdsCluster(routeName string, destinations []*ir.RouteDestination) (*cl // load balancers need the value to be set. LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}} localities = append(localities, locality) - clusterName := getXdsClusterName(routeName) + clusterName := routeName return &cluster.Cluster{ Name: clusterName, ConnectTimeout: durationpb.New(5 * time.Second), diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index e3ea4c82fe8..61bdf557279 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -16,25 +16,44 @@ import ( "github.com/envoyproxy/gateway/internal/ir" ) -func buildXdsListener(httpListener *ir.HTTPListener) (*listener.Listener, error) { - if httpListener == nil { - return nil, errors.New("http listener is nil") +func buildXdsListener(name, address string, port uint32) *listener.Listener { + return &listener.Listener{ + Name: name, + Address: &core.Address{ + Address: &core.Address_SocketAddress{ + SocketAddress: &core.SocketAddress{ + Protocol: core.SocketAddress_TCP, + Address: address, + PortSpecifier: &core.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, } +} +func addXdsHTTPFilterChain(xdsListener *listener.Listener, irListener *ir.HTTPListener) error { routerAny, err := anypb.New(&router.Router{}) if err != nil { - return nil, err + return err } // HTTP filter configuration + var statPrefix string + if irListener.TLS != nil { + statPrefix = "https" + } else { + statPrefix = "http" + } mgr := &hcm.HttpConnectionManager{ CodecType: hcm.HttpConnectionManager_AUTO, - StatPrefix: "http", + StatPrefix: statPrefix, RouteSpecifier: &hcm.HttpConnectionManager_Rds{ Rds: &hcm.Rds{ ConfigSource: makeConfigSource(), // Configure route name to be found via RDS. - RouteConfigName: getXdsRouteName(httpListener.Name), + RouteConfigName: irListener.Name, }, }, // Use only router. @@ -46,40 +65,67 @@ func buildXdsListener(httpListener *ir.HTTPListener) (*listener.Listener, error) mgrAny, err := anypb.New(mgr) if err != nil { - return nil, err + return err } - return &listener.Listener{ - Name: getXdsListenerName(httpListener.Name, httpListener.Port), - Address: &core.Address{ - Address: &core.Address_SocketAddress{ - SocketAddress: &core.SocketAddress{ - Protocol: core.SocketAddress_TCP, - Address: httpListener.Address, - PortSpecifier: &core.SocketAddress_PortValue{ - PortValue: httpListener.Port, - }, - }, + filterChain := &listener.FilterChain{ + Filters: []*listener.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &listener.Filter_TypedConfig{ + TypedConfig: mgrAny, }, - }, - FilterChains: []*listener.FilterChain{{ - Filters: []*listener.Filter{{ - Name: wellknown.HTTPConnectionManager, - ConfigType: &listener.Filter_TypedConfig{ - TypedConfig: mgrAny, - }, - }}, }}, - }, nil + } + + if irListener.TLS != nil { + tSocket, err := buildXdsDownstreamTLSSocket(irListener.Name, irListener.TLS) + if err != nil { + return err + } + filterChain.TransportSocket = tSocket + filterChain.FilterChainMatch = &listener.FilterChainMatch{ + ServerNames: irListener.Hostnames, + } + + if err := addXdsTLSInspectorFilter(xdsListener); err != nil { + return err + } + + xdsListener.FilterChains = append(xdsListener.FilterChains, filterChain) + } else { + // Add the HTTP filter chain as the default filter chain + // Make sure one does not exist + if xdsListener.DefaultFilterChain != nil { + return errors.New("default filter chain already exists") + } + xdsListener.DefaultFilterChain = filterChain + } + + return nil +} + +// findXdsHTTPRouteConfigName finds the name of the route config associated with the +// http connection manager within the default filter chain. +func findXdsHTTPRouteConfigName(xdsListener *listener.Listener) (string, error) { + for _, filter := range xdsListener.DefaultFilterChain.Filters { + if filter.Name == wellknown.HTTPConnectionManager { + m := new(hcm.HttpConnectionManager) + if err := filter.GetTypedConfig().UnmarshalTo(m); err != nil { + return "", err + } + return m.GetRds().GetRouteConfigName(), nil + } + } + return "", errors.New("unable to find route config") } -func buildXdsTCPListener(clusterName string, tcpListener *ir.TCPListener) (*listener.Listener, error) { - if tcpListener == nil { - return nil, errors.New("http listener is nil") +func addXdsTCPFilterChain(xdsListener *listener.Listener, irListener *ir.TCPListener, clusterName string) error { + if irListener == nil { + return errors.New("tcp listener is nil") } statPrefix := "tcp" - if tcpListener.TLS != nil { + if irListener.TLS != nil { statPrefix = "passthrough" } mgr := &tcp.TcpProxy{ @@ -90,7 +136,7 @@ func buildXdsTCPListener(clusterName string, tcpListener *ir.TCPListener) (*list } mgrAny, err := anypb.New(mgr) if err != nil { - return nil, err + return err } filterChain := &listener.FilterChain{ @@ -101,44 +147,48 @@ func buildXdsTCPListener(clusterName string, tcpListener *ir.TCPListener) (*list }, }}, } - if tcpListener.TLS != nil { + + if irListener.TLS != nil { filterChain.FilterChainMatch = &listener.FilterChainMatch{ - ServerNames: tcpListener.TLS.SNIs, + ServerNames: irListener.TLS.SNIs, + } + + if err := addXdsTLSInspectorFilter(xdsListener); err != nil { + return err } - } - xdsListener := &listener.Listener{ - Name: getXdsListenerName(tcpListener.Name, tcpListener.Port), - Address: &core.Address{ - Address: &core.Address_SocketAddress{ - SocketAddress: &core.SocketAddress{ - Protocol: core.SocketAddress_TCP, - Address: tcpListener.Address, - PortSpecifier: &core.SocketAddress_PortValue{ - PortValue: tcpListener.Port, - }, - }, - }, - }, - FilterChains: []*listener.FilterChain{filterChain}, } - if tcpListener.TLS != nil { - tlsInspector := &tls_inspector.TlsInspector{} - tlsInspectorAny, err := anypb.New(tlsInspector) - if err != nil { - return nil, err + xdsListener.FilterChains = append(xdsListener.FilterChains, filterChain) + + return nil +} + +// addXdsTLSInspectorFilter adds a Tls Inspector filter if it does not yet exist. +func addXdsTLSInspectorFilter(xdsListener *listener.Listener) error { + // Return early if it exists + for _, filter := range xdsListener.ListenerFilters { + if filter.Name == wellknown.TlsInspector { + return nil } + } - xdsListener.ListenerFilters = []*listener.ListenerFilter{{ - Name: wellknown.TlsInspector, - ConfigType: &listener.ListenerFilter_TypedConfig{ - TypedConfig: tlsInspectorAny, - }, - }} + tlsInspector := &tls_inspector.TlsInspector{} + tlsInspectorAny, err := anypb.New(tlsInspector) + if err != nil { + return err } - return xdsListener, nil + filter := &listener.ListenerFilter{ + Name: wellknown.TlsInspector, + ConfigType: &listener.ListenerFilter_TypedConfig{ + TypedConfig: tlsInspectorAny, + }, + } + + xdsListener.ListenerFilters = append(xdsListener.ListenerFilters, filter) + + return nil } func buildXdsDownstreamTLSSocket(listenerName string, @@ -148,7 +198,7 @@ func buildXdsDownstreamTLSSocket(listenerName string, TlsCertificateSdsSecretConfigs: []*tls.SdsSecretConfig{{ // Generate key name for this listener. The actual key will be // delivered to Envoy via SDS. - Name: getXdsSecretName(listenerName), + Name: listenerName, SdsConfig: makeConfigSource(), }}, }, @@ -171,7 +221,7 @@ func buildXdsDownstreamTLSSecret(listenerName string, tlsConfig *ir.TLSListenerConfig) (*tls.Secret, error) { // Build the tls secret return &tls.Secret{ - Name: getXdsSecretName(listenerName), + Name: listenerName, Type: &tls.Secret_TlsCertificate{ TlsCertificate: &tls.TlsCertificate{ CertificateChain: &core.DataSource{ diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index d231a606971..dc5acd70e4b 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -135,7 +135,7 @@ func buildXdsStringMatcher(irMatch *ir.StringMatch) *matcher.StringMatcher { func buildXdsRouteAction(routeName string) *route.RouteAction { return &route.RouteAction{ ClusterSpecifier: &route.RouteAction_Cluster{ - Cluster: getXdsClusterName(routeName), + Cluster: routeName, }, } } @@ -148,7 +148,7 @@ func buildXdsWeightedRouteAction(httpRoute *ir.HTTPRoute) *route.RouteAction { Weight: &wrapperspb.UInt32Value{Value: httpRoute.BackendWeights.Invalid}, }, { - Name: getXdsClusterName(httpRoute.Name), + Name: httpRoute.Name, Weight: &wrapperspb.UInt32Value{Value: httpRoute.BackendWeights.Valid}, }, } diff --git a/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml b/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml new file mode 100644 index 00000000000..431eb838f16 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml @@ -0,0 +1,66 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "example.com" + routes: + - name: "first-route" + destinations: + - host: "1.2.3.4" + port: 50000 +- name: "second-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "example.net" + routes: + - name: "second-route" + destinations: + - host: "1.2.3.4" + port: 50000 +- name: "third-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "foo.com" + tls: + serverCertificate: [99, 101, 114, 116, 45, 100, 97, 116, 97] # byte slice representation of "cert-data" + privateKey: [107, 101, 121, 45, 100, 97, 116, 97] # byte slice representation of "key-data" + routes: + - name: "third-route" + destinations: + - host: "1.2.3.4" + port: 50000 +- name: "fourth-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "foo.net" + tls: + serverCertificate: [99, 101, 114, 116, 45, 100, 97, 116, 97] # byte slice representation of "cert-data" + privateKey: [107, 101, 121, 45, 100, 97, 116, 97] # byte slice representation of "key-data" + routes: + - name: "fourth-route" + destinations: + - host: "1.2.3.4" + port: 50000 +tcp: +- name: "fifth-listener" + address: "0.0.0.0" + port: 10080 + tls: + snis: + - bar.com + destinations: + - host: "1.2.3.4" + port: 50000 +- name: "sixth-listener" + address: "0.0.0.0" + port: 10080 + tls: + snis: + - bar.net + destinations: + - host: "1.2.3.4" + port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.clusters.yaml index c517dcb5690..0a8b9dd797d 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_direct-route + clusterName: direct-route endpoints: - lbEndpoints: - endpoint: @@ -13,6 +13,6 @@ portValue: 50000 loadBalancingWeight: 1 locality: {} - name: cluster_direct-route + name: direct-route outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml index f1077139a82..2f73c8e922c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml @@ -2,8 +2,8 @@ socketAddress: address: 0.0.0.0 portValue: 10080 - filterChains: - - filters: + defaultFilterChain: + filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -21,6 +21,6 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: route_first-listener + routeConfigName: first-listener statPrefix: http - name: listener_first-listener_10080 + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.routes.yaml index e59392e464d..59bf816dd1b 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.routes.yaml @@ -1,8 +1,8 @@ -- name: route_first-listener +- name: first-listener virtualHosts: - domains: - '*' - name: route_first-listener + name: first-listener routes: - directResponse: body: diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.clusters.yaml index da7827f6088..3709f73545d 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_redirect-route + clusterName: redirect-route endpoints: - lbEndpoints: - endpoint: @@ -13,6 +13,6 @@ portValue: 50000 loadBalancingWeight: 1 locality: {} - name: cluster_redirect-route + name: redirect-route outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml index f1077139a82..2f73c8e922c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml @@ -2,8 +2,8 @@ socketAddress: address: 0.0.0.0 portValue: 10080 - filterChains: - - filters: + defaultFilterChain: + filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -21,6 +21,6 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: route_first-listener + routeConfigName: first-listener statPrefix: http - name: listener_first-listener_10080 + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.routes.yaml index 2c2f4436a9c..462febfae07 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.routes.yaml @@ -1,8 +1,8 @@ -- name: route_first-listener +- name: first-listener virtualHosts: - domains: - '*' - name: route_first-listener + name: first-listener routes: - match: prefix: / diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.clusters.yaml index ce7adf442e6..035a07ecd59 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_request-header-route + clusterName: request-header-route endpoints: - lbEndpoints: - endpoint: @@ -13,6 +13,6 @@ portValue: 50000 loadBalancingWeight: 1 locality: {} - name: cluster_request-header-route + name: request-header-route outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml index f1077139a82..2f73c8e922c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml @@ -2,8 +2,8 @@ socketAddress: address: 0.0.0.0 portValue: 10080 - filterChains: - - filters: + defaultFilterChain: + filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -21,6 +21,6 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: route_first-listener + routeConfigName: first-listener statPrefix: http - name: listener_first-listener_10080 + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.routes.yaml index 5d9c5f08a63..0284c21fc0d 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.routes.yaml @@ -1,8 +1,8 @@ -- name: route_first-listener +- name: first-listener virtualHosts: - domains: - '*' - name: route_first-listener + name: first-listener routes: - match: prefix: / @@ -31,4 +31,4 @@ - some-header5 - some-header6 route: - cluster: cluster_request-header-route + cluster: request-header-route diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml index c65cb16a6a4..c10babdc70f 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_first-route + clusterName: first-route endpoints: - lbEndpoints: - endpoint: @@ -13,6 +13,6 @@ portValue: 50000 loadBalancingWeight: 1 locality: {} - name: cluster_first-route + name: first-route outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml index f1077139a82..2f73c8e922c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml @@ -2,8 +2,8 @@ socketAddress: address: 0.0.0.0 portValue: 10080 - filterChains: - - filters: + defaultFilterChain: + filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -21,6 +21,6 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: route_first-listener + routeConfigName: first-listener statPrefix: http - name: listener_first-listener_10080 + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml index 7b13acc60a9..d8966acfc10 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml @@ -1,8 +1,8 @@ -- name: route_first-listener +- name: first-listener virtualHosts: - domains: - '*' - name: route_first-listener + name: first-listener routes: - match: prefix: / @@ -12,6 +12,6 @@ clusters: - name: invalid-backend-cluster weight: 1 - - name: cluster_first-route + - name: first-route weight: 1 totalWeight: 2 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route.clusters.yaml index c65cb16a6a4..c10babdc70f 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_first-route + clusterName: first-route endpoints: - lbEndpoints: - endpoint: @@ -13,6 +13,6 @@ portValue: 50000 loadBalancingWeight: 1 locality: {} - name: cluster_first-route + name: first-route outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml index f1077139a82..2f73c8e922c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml @@ -2,8 +2,8 @@ socketAddress: address: 0.0.0.0 portValue: 10080 - filterChains: - - filters: + defaultFilterChain: + filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -21,6 +21,6 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: route_first-listener + routeConfigName: first-listener statPrefix: http - name: listener_first-listener_10080 + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml index a550146b227..ed122e552aa 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml @@ -1,10 +1,10 @@ -- name: route_first-listener +- name: first-listener virtualHosts: - domains: - '*' - name: route_first-listener + name: first-listener routes: - match: prefix: / route: - cluster: cluster_first-route + cluster: first-route diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.clusters.yaml new file mode 100644 index 00000000000..e4e723e49f4 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.clusters.yaml @@ -0,0 +1,102 @@ +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: first-route + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + locality: {} + name: first-route + outlierDetection: {} + type: STATIC +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: second-route + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + locality: {} + name: second-route + outlierDetection: {} + type: STATIC +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: third-route + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + locality: {} + name: third-route + outlierDetection: {} + type: STATIC +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: fourth-route + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + locality: {} + name: fourth-route + outlierDetection: {} + type: STATIC +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: fifth-listener + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + locality: {} + name: fifth-listener + outlierDetection: {} + type: STATIC +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: sixth-listener + endpoints: + - loadBalancingWeight: 1 + locality: {} + name: sixth-listener + outlierDetection: {} + type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml new file mode 100644 index 00000000000..5732ab6b533 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml @@ -0,0 +1,127 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + rds: + configSource: + apiConfigSource: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: xds_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + resourceApiVersion: V3 + routeConfigName: first-listener + statPrefix: http + filterChains: + - filterChainMatch: + serverNames: + - foo.com + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + rds: + configSource: + apiConfigSource: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: xds_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + resourceApiVersion: V3 + routeConfigName: third-listener + statPrefix: https + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: third-listener + sdsConfig: + apiConfigSource: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: xds_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + resourceApiVersion: V3 + - filterChainMatch: + serverNames: + - foo.net + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + rds: + configSource: + apiConfigSource: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: xds_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + resourceApiVersion: V3 + routeConfigName: fourth-listener + statPrefix: https + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: fourth-listener + sdsConfig: + apiConfigSource: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: xds_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + resourceApiVersion: V3 + - filterChainMatch: + serverNames: + - bar.com + filters: + - name: envoy.filters.network.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: fifth-listener + statPrefix: passthrough + - filterChainMatch: + serverNames: + - bar.net + filters: + - name: envoy.filters.network.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: sixth-listener + statPrefix: passthrough + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml new file mode 100644 index 00000000000..d5b2375a592 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml @@ -0,0 +1,38 @@ +- name: first-listener + virtualHosts: + - domains: + - example.com + name: first-listener + routes: + - match: + prefix: / + route: + cluster: first-route + - domains: + - example.net + name: second-listener + routes: + - match: + prefix: / + route: + cluster: second-route +- name: third-listener + virtualHosts: + - domains: + - foo.com + name: third-listener + routes: + - match: + prefix: / + route: + cluster: third-route +- name: fourth-listener + virtualHosts: + - domains: + - foo.net + name: fourth-listener + routes: + - match: + prefix: / + route: + cluster: fourth-route diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml new file mode 100644 index 00000000000..0464fffd2c7 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml @@ -0,0 +1,12 @@ +- name: third-listener + tlsCertificate: + certificateChain: + inlineBytes: Y2VydC1kYXRh + privateKey: + inlineBytes: a2V5LWRhdGE= +- name: fourth-listener + tlsCertificate: + certificateChain: + inlineBytes: Y2VydC1kYXRh + privateKey: + inlineBytes: a2V5LWRhdGE= diff --git a/internal/xds/translator/testdata/out/xds-ir/simple-tls.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/simple-tls.clusters.yaml index c65cb16a6a4..c10babdc70f 100644 --- a/internal/xds/translator/testdata/out/xds-ir/simple-tls.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/simple-tls.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_first-route + clusterName: first-route endpoints: - lbEndpoints: - endpoint: @@ -13,6 +13,6 @@ portValue: 50000 loadBalancingWeight: 1 locality: {} - name: cluster_first-route + name: first-route outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml index 58110f0b99d..4e05738e9b5 100644 --- a/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml @@ -3,7 +3,10 @@ address: 0.0.0.0 portValue: 10080 filterChains: - - filters: + - filterChainMatch: + serverNames: + - '*' + filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -21,15 +24,15 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: route_first-listener - statPrefix: http + routeConfigName: first-listener + statPrefix: https transportSocket: name: envoy.transport_sockets.tls typedConfig: '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext commonTlsContext: tlsCertificateSdsSecretConfigs: - - name: secret_first-listener + - name: first-listener sdsConfig: apiConfigSource: apiType: DELTA_GRPC @@ -39,4 +42,8 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - name: listener_first-listener_10080 + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/simple-tls.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/simple-tls.routes.yaml index a550146b227..ed122e552aa 100644 --- a/internal/xds/translator/testdata/out/xds-ir/simple-tls.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/simple-tls.routes.yaml @@ -1,10 +1,10 @@ -- name: route_first-listener +- name: first-listener virtualHosts: - domains: - '*' - name: route_first-listener + name: first-listener routes: - match: prefix: / route: - cluster: cluster_first-route + cluster: first-route diff --git a/internal/xds/translator/testdata/out/xds-ir/simple-tls.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/simple-tls.secrets.yaml index 23859a66abf..9155e5c882d 100644 --- a/internal/xds/translator/testdata/out/xds-ir/simple-tls.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/simple-tls.secrets.yaml @@ -1,4 +1,4 @@ -- name: secret_first-listener +- name: first-listener tlsCertificate: certificateChain: inlineBytes: Y2VydC1kYXRh diff --git a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml index 95ed98685be..48ee1e2ba40 100644 --- a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.clusters.yaml @@ -3,7 +3,7 @@ connectTimeout: 5s dnsLookupFamily: V4_ONLY loadAssignment: - clusterName: cluster_tls-passthrough + clusterName: tls-passthrough endpoints: - lbEndpoints: - endpoint: @@ -18,6 +18,6 @@ portValue: 50001 loadBalancingWeight: 1 locality: {} - name: cluster_tls-passthrough + name: tls-passthrough outlierDetection: {} type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml index b52cb5d8d2b..53774a46925 100644 --- a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml @@ -10,10 +10,10 @@ - name: envoy.filters.network.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - cluster: cluster_tls-passthrough + cluster: tls-passthrough statPrefix: passthrough listenerFilters: - name: envoy.filters.listener.tls_inspector typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector - name: listener_tls-passthrough_10080 + name: tls-passthrough diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 434b0e62e8b..810ebe00d64 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -2,9 +2,9 @@ package translator import ( "errors" - "fmt" core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/tetratelabs/multierror" @@ -22,21 +22,48 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { tCtx := new(types.ResourceVersionTable) for _, httpListener := range ir.HTTP { - // 1:1 between IR HTTPListener and xDS Listener - xdsListener, err := buildXdsListener(httpListener) - if err != nil { - return nil, multierror.Append(err, errors.New("error building xds listener")) + addFilterChain := true + var xdsRouteCfg *route.RouteConfiguration + + // Search for an existing listener, if it does not exist, create one. + xdsListener := findXdsListener(tCtx, httpListener.Address, httpListener.Port) + if xdsListener == nil { + xdsListener = buildXdsListener(httpListener.Name, httpListener.Address, httpListener.Port) + tCtx.AddXdsResource(resource.ListenerType, xdsListener) + } else { + // If an existing listener exists, dont create a new filter chain + // for HTTP traffic, match on the Domains field within VirtualHosts + // within the same RouteConfiguration instead + if httpListener.TLS == nil { + addFilterChain = false + } + // Find the route config associated with this listener that + // maps to the filter chain for http traffic + // There should only be one of these per xds listener + routeName, err := findXdsHTTPRouteConfigName(xdsListener) + if err != nil { + return nil, err + } + xdsRouteCfg = findXdsRouteConfig(tCtx, routeName) + if xdsRouteCfg == nil { + return nil, errors.New("unable to find xds route config") + } } - // 1:1 between IR TLSListenerConfig and xDS Secret - if httpListener.TLS != nil { - // Build downstream TLS details. - tSocket, err := buildXdsDownstreamTLSSocket(httpListener.Name, httpListener.TLS) - if err != nil { - return nil, multierror.Append(err, errors.New("error building xds listener tls socket")) + if addFilterChain { + if err := addXdsHTTPFilterChain(xdsListener, httpListener); err != nil { + return nil, err + } + + xdsRouteCfg = &route.RouteConfiguration{ + Name: httpListener.Name, } - xdsListener.FilterChains[0].TransportSocket = tSocket + tCtx.AddXdsResource(resource.RouteType, xdsRouteCfg) + } + + // 1:1 between IR TLSListenerConfig and xDS Secret + if httpListener.TLS != nil { secret, err := buildXdsDownstreamTLSSecret(httpListener.Name, httpListener.TLS) if err != nil { return nil, multierror.Append(err, errors.New("error building xds listener tls secret")) @@ -46,9 +73,8 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { // Allocate virtual host for this httpListener. // 1:1 between IR HTTPListener and xDS VirtualHost - routeName := getXdsRouteName(httpListener.Name) vHost := &route.VirtualHost{ - Name: routeName, + Name: httpListener.Name, Domains: httpListener.Hostnames, } @@ -72,13 +98,7 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { } - xdsRouteCfg := &route.RouteConfiguration{ - Name: routeName, - } xdsRouteCfg.VirtualHosts = append(xdsRouteCfg.VirtualHosts, vHost) - - tCtx.AddXdsResource(resource.ListenerType, xdsListener) - tCtx.AddXdsResource(resource.RouteType, xdsRouteCfg) } for _, tcpListener := range ir.TCP { @@ -89,31 +109,51 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { } tCtx.AddXdsResource(resource.ClusterType, xdsCluster) - // 1:1 between IR TCPListener and xDS Listener - xdsListener, err := buildXdsTCPListener(xdsCluster.Name, tcpListener) - if err != nil { - return nil, multierror.Append(err, errors.New("error building xds listener")) + // Search for an existing listener, if it does not exist, create one. + xdsListener := findXdsListener(tCtx, tcpListener.Address, tcpListener.Port) + if xdsListener == nil { + xdsListener = buildXdsListener(tcpListener.Name, tcpListener.Address, tcpListener.Port) + tCtx.AddXdsResource(resource.ListenerType, xdsListener) } - tCtx.AddXdsResource(resource.ListenerType, xdsListener) + if err := addXdsTCPFilterChain(xdsListener, tcpListener, xdsCluster.Name); err != nil { + return nil, err + } } return tCtx, nil } -func getXdsRouteName(listenerName string) string { - return fmt.Sprintf("route_%s", listenerName) -} +// findXdsListener finds an xds listener with the same address and port, and returns nil if there is no match. +func findXdsListener(tCtx *types.ResourceVersionTable, address string, port uint32) *listener.Listener { + if tCtx == nil || tCtx.XdsResources == nil || tCtx.XdsResources[resource.ListenerType] == nil { + return nil + } -func getXdsListenerName(listenerName string, listenerPort uint32) string { - return fmt.Sprintf("listener_%s_%d", listenerName, listenerPort) -} + for _, r := range tCtx.XdsResources[resource.ListenerType] { + listener := r.(*listener.Listener) + addr := listener.GetAddress() + if addr.GetSocketAddress().GetPortValue() == port && addr.GetSocketAddress().Address == address { + return listener + } + } -func getXdsSecretName(listenerName string) string { - return fmt.Sprintf("secret_%s", listenerName) + return nil } -func getXdsClusterName(routeName string) string { - return fmt.Sprintf("cluster_%s", routeName) +// findXdsRouteConfig finds an xds route with the name and returns nil if there is no match. +func findXdsRouteConfig(tCtx *types.ResourceVersionTable, name string) *route.RouteConfiguration { + if tCtx == nil || tCtx.XdsResources == nil || tCtx.XdsResources[resource.RouteType] == nil { + return nil + } + + for _, r := range tCtx.XdsResources[resource.RouteType] { + route := r.(*route.RouteConfiguration) + if route.Name == name { + return route + } + } + + return nil } // Point to xds cluster. diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 0102b6d84f7..c968badfa80 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -53,6 +53,10 @@ func TestTranslate(t *testing.T) { { name: "tls-route-passthrough", }, + { + name: "multiple-listeners-same-port", + requireSecrets: true, + }, } for _, tc := range testCases { diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index b50613936b9..b3b748ca43f 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -58,6 +58,8 @@ func TestGatewayAPIConformance(t *testing.T) { tests.HTTPRouteCrossNamespace, tests.HTTPRouteHeaderMatching, tests.HTTPRouteMatchingAcrossRoutes, + tests.HTTPRouteHostnameIntersection, + tests.HTTPRouteListenerHostnameMatching, tests.HTTPRouteInvalidNonExistentBackendRef, tests.HTTPRouteInvalidBackendRefUnknownKind, tests.HTTPRouteInvalidCrossNamespaceBackendRef, From 36160071bd358e7e6ddd3fe06ed2ed0558b81b8a Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Wed, 19 Oct 2022 00:35:39 +0800 Subject: [PATCH 041/113] update docs.md with Markdown support (#589) Signed-off-by: zhaohuabing --- DOCS.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/DOCS.md b/DOCS.md index 4f9eb8a412d..643636121bd 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1,8 +1,9 @@ # Working on the Envoy Gateway Docs -The documentation for the Envoy Gateway lives in the `docs/` directory. It's -written using [reStructuredText], so you'll need a working familiarity with -that! +The documentation for the Envoy Gateway lives in the `docs/` directory. Any +individual document can be written using either [reStructuredText] or [Markdown], +you can choose the format that you're most comfortable with when working on the +documentation. ## Documentation Structure @@ -15,7 +16,8 @@ to be in `docs/index.rst`'s `toctree` though. ## Documentation Workflow -To work with the docs, just edit reStructuredText files in `docs`, then run +To work with the docs, just edit reStructuredText or Markdown files in `docs`, +then run ```bash make docs @@ -29,15 +31,11 @@ either simply by pointing a web browser at the `file://` path to your cd docs/html ; python3 -m http.server ``` -### What About Markdown Files? - -There are currently still some pre-RST Markdown files in the `docs/` directory. -Those should be turned into RST files and brought into the brave new world. - ## Publishing Docs Whenever docs are pushed to `main`, CI will publish the built docs to GitHub Pages. For more details, see `.github/workflows/docs.yaml`. [reStructuredText]: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html +[Markdown]: https://daringfireball.net/projects/markdown/syntax From 9ab00f2621a7e7a5ac3d29d81f55e0bafb670113 Mon Sep 17 00:00:00 2001 From: Shubham Chauhan Date: Tue, 18 Oct 2022 22:45:41 +0530 Subject: [PATCH 042/113] Fix Service object remove from resource map from route controller (#549) * Fix Service object remove from resource map from route controller This commit fetches the services that were referenced by the route which has now been deleted, and removes those objects from the resource map. In order to facilitate this, the commit introduces a provider cache for kubernetes that stores mappings between kubernetes objects - for now it stores the map between [tls/http]route -> backend service references. So that once the [tls/http]route is deleted we have the services that it referenced. Signed-off-by: Shubham Chauhan --- internal/provider/kubernetes/httproute.go | 29 +++++++-- internal/provider/kubernetes/kubernetes.go | 8 ++- internal/provider/kubernetes/store.go | 72 ++++++++++++++++++++++ internal/provider/kubernetes/store_test.go | 62 +++++++++++++++++++ internal/provider/kubernetes/tlsroute.go | 39 +++++++++--- 5 files changed, 193 insertions(+), 17 deletions(-) create mode 100644 internal/provider/kubernetes/store.go create mode 100644 internal/provider/kubernetes/store_test.go diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go index e01a77a1924..b1ce4cd0e19 100644 --- a/internal/provider/kubernetes/httproute.go +++ b/internal/provider/kubernetes/httproute.go @@ -28,6 +28,8 @@ import ( ) const ( + kindHTTPRoute = "HTTPRoute" + serviceHTTPRouteIndex = "serviceHTTPRouteBackendRef" ) @@ -37,18 +39,20 @@ type httpRouteReconciler struct { statusUpdater status.Updater classController gwapiv1b1.GatewayController - resources *message.ProviderResources + resources *message.ProviderResources + referenceStore *providerReferenceStore } // newHTTPRouteController creates the httproute controller from mgr. The controller will be pre-configured // to watch for HTTPRoute objects across all namespaces. -func newHTTPRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources) error { +func newHTTPRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources, referenceStore *providerReferenceStore) error { r := &httpRouteReconciler{ client: mgr.GetClient(), log: cfg.Logger, classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), statusUpdater: su, resources: resources, + referenceStore: referenceStore, } c, err := controller.New("httproute", mgr, controller.Options{Reconciler: r}) @@ -254,6 +258,10 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R // the resource map if it exists. if _, ok := r.resources.Services.Load(svcKey); ok { r.resources.Services.Delete(svcKey) + r.referenceStore.removeRouteToServicesMapping( + ObjectKindNamespacedName{kindHTTPRoute, route.Namespace, route.Name}, + svcKey, + ) log.Info("deleted service from resource map") } } @@ -263,6 +271,10 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R // The backendRef Service exists, so add it to the resource map. r.resources.Services.Store(svcKey, svc) + r.referenceStore.updateRouteToServicesMapping( + ObjectKindNamespacedName{kindHTTPRoute, route.Namespace, route.Name}, + svcKey, + ) log.Info("added service to resource map") } } @@ -282,8 +294,17 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R if !found { r.resources.Namespaces.Delete(request.Namespace) log.Info("deleted namespace from resource map") - r.resources.Services.Delete(request.NamespacedName) - log.Info("deleted service from resource map") + } + + // Delete the Service from the resource maps if no other + // routes (TLSRoute or HTTPRoute) reference that Service. + routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, request.Namespace, request.Name}) + for svc := range routeServices { + r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, request.Namespace, request.Name}, svc) + if r.referenceStore.isServiceReferredByRoutes(svc) { + r.resources.Services.Delete(request.NamespacedName) + log.Info("deleted service from resource map") + } } } diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index 9c265153329..cc5670a90b6 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -45,6 +45,9 @@ func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResour return nil, fmt.Errorf("failed to add status update handler %v", err) } + // Initialize kubernetes provider referenceStore to store additional object mappings. + referenceStore := newProviderReferenceStore() + // Create and register the controllers with the manager. if err := newGatewayClassController(mgr, svr, updateHandler.Writer(), resources); err != nil { return nil, fmt.Errorf("failed to create gatewayclass controller: %w", err) @@ -53,11 +56,10 @@ func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResour return nil, fmt.Errorf("failed to create gateway controller: %w", err) } - if err := newHTTPRouteController(mgr, svr, updateHandler.Writer(), resources); err != nil { + if err := newHTTPRouteController(mgr, svr, updateHandler.Writer(), resources, referenceStore); err != nil { return nil, fmt.Errorf("failed to create httproute controller: %w", err) } - - if err := newTLSRouteController(mgr, svr, updateHandler.Writer(), resources); err != nil { + if err := newTLSRouteController(mgr, svr, updateHandler.Writer(), resources, referenceStore); err != nil { return nil, fmt.Errorf("failed to create tlsroute controller: %w", err) } diff --git a/internal/provider/kubernetes/store.go b/internal/provider/kubernetes/store.go new file mode 100644 index 00000000000..8d9be70e5c4 --- /dev/null +++ b/internal/provider/kubernetes/store.go @@ -0,0 +1,72 @@ +package kubernetes + +import ( + "sync" + + "k8s.io/apimachinery/pkg/types" +) + +// providerReferenceStore maintains additional mappings related to Kubernetes provider +// resources. The mappings are regularly updated from the reconcilers based +// on the existence of the object in the Kubernetes datastore. +type providerReferenceStore struct { + mu sync.Mutex + + // routeToServicesMappings maintains a mapping of a Route object, + // and the Services it references. For instance + // HTTPRoute/ns1/route1 -> { ns1/svc1, ns1/svc2, ns2/svc1 } + // TLSRoute/ns1/route1 -> { ns1/svc1, ns2/svc2 } + routeToServicesMappings map[ObjectKindNamespacedName]map[types.NamespacedName]struct{} +} + +type ObjectKindNamespacedName struct { + kind string + namespace string + name string +} + +func newProviderReferenceStore() *providerReferenceStore { + return &providerReferenceStore{ + routeToServicesMappings: map[ObjectKindNamespacedName]map[types.NamespacedName]struct{}{}, + } +} + +func (p *providerReferenceStore) getRouteToServicesMapping(route ObjectKindNamespacedName) map[types.NamespacedName]struct{} { + p.mu.Lock() + defer p.mu.Unlock() + + return p.routeToServicesMappings[route] +} + +func (p *providerReferenceStore) updateRouteToServicesMapping(route ObjectKindNamespacedName, service types.NamespacedName) { + p.mu.Lock() + defer p.mu.Unlock() + + if len(p.routeToServicesMappings[route]) == 0 { + p.routeToServicesMappings[route] = map[types.NamespacedName]struct{}{service: {}} + } else { + p.routeToServicesMappings[route][service] = struct{}{} + } +} + +func (p *providerReferenceStore) removeRouteToServicesMapping(route ObjectKindNamespacedName, service types.NamespacedName) { + p.mu.Lock() + defer p.mu.Unlock() + + delete(p.routeToServicesMappings[route], service) + if len(p.routeToServicesMappings[route]) == 0 { + delete(p.routeToServicesMappings, route) + } +} + +func (p *providerReferenceStore) isServiceReferredByRoutes(service types.NamespacedName) bool { + p.mu.Lock() + defer p.mu.Unlock() + + for _, svcs := range p.routeToServicesMappings { + if _, ok := svcs[service]; ok { + return true + } + } + return false +} diff --git a/internal/provider/kubernetes/store_test.go b/internal/provider/kubernetes/store_test.go new file mode 100644 index 00000000000..ca5d17c6ad4 --- /dev/null +++ b/internal/provider/kubernetes/store_test.go @@ -0,0 +1,62 @@ +package kubernetes + +import ( + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" +) + +func TestProviderReferenceStore(t *testing.T) { + cache := newProviderReferenceStore() + + testCases := []struct { + name string + test func(t *testing.T, c *providerReferenceStore) + }{ + { + name: "route to service mappings", + test: testRouteToServicesMappings, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + tc.test(t, cache) + }) + } +} + +func testRouteToServicesMappings(t *testing.T, cache *providerReferenceStore) { + httpr1 := ObjectKindNamespacedName{"HTTPRoute", "ns1", "r1"} + tlsr1 := ObjectKindNamespacedName{"TLSRoute", "ns1", "r1"} + + ns1svc1 := types.NamespacedName{Namespace: "ns1", Name: "svc1"} + ns1svc2 := types.NamespacedName{Namespace: "ns1", Name: "svc2"} + + // Add HTTPRoute/ns1/r1 -> ns1/svc1 mapping + cache.updateRouteToServicesMapping(httpr1, ns1svc1) + require.Equal(t, map[types.NamespacedName]struct{}{ns1svc1: {}}, cache.getRouteToServicesMapping(httpr1)) + + // Add HTTPRoute/ns1/r1 -> ns1/svc2 mapping + // Add TLSRoute/ns1/r1 -> ns1/svc2 mapping + cache.updateRouteToServicesMapping(tlsr1, ns1svc2) + cache.updateRouteToServicesMapping(httpr1, ns1svc2) + require.Equal(t, map[types.NamespacedName]struct{}{ns1svc2: {}}, cache.getRouteToServicesMapping(tlsr1)) + require.Equal(t, map[types.NamespacedName]struct{}{ns1svc1: {}, ns1svc2: {}}, cache.getRouteToServicesMapping(httpr1)) + + // Remove HTTPRoute/ns1/r1 -> ns1/svc1 mapping + cache.removeRouteToServicesMapping(httpr1, ns1svc1) + require.Equal(t, map[types.NamespacedName]struct{}{ns1svc2: {}}, cache.getRouteToServicesMapping(httpr1)) + + // Remove TLSRoute/ns1/r1 -> ns1/svc2 mapping + cache.removeRouteToServicesMapping(tlsr1, ns1svc2) + require.Equal(t, map[types.NamespacedName]struct{}(nil), cache.getRouteToServicesMapping(tlsr1)) + + // Verify that ns1svc2 is still referred by another route (HTTPRoute/ns1/r1) + require.Equal(t, true, cache.isServiceReferredByRoutes(ns1svc2)) + + // Verify that ns1svc1 is not referred by anu other route + require.Equal(t, false, cache.isServiceReferredByRoutes(ns1svc1)) +} diff --git a/internal/provider/kubernetes/tlsroute.go b/internal/provider/kubernetes/tlsroute.go index f78a661b1d8..7aee3b0e39c 100644 --- a/internal/provider/kubernetes/tlsroute.go +++ b/internal/provider/kubernetes/tlsroute.go @@ -29,6 +29,8 @@ import ( ) const ( + kindTLSRoute = "TLSRoute" + serviceTLSRouteIndex = "serviceTLSRouteBackendRef" ) @@ -38,18 +40,20 @@ type tlsRouteReconciler struct { statusUpdater status.Updater classController gwapiv1b1.GatewayController - resources *message.ProviderResources + resources *message.ProviderResources + referenceStore *providerReferenceStore } // newTLSRouteController creates the tlsroute controller from mgr. The controller will be pre-configured // to watch for TLSRoute objects across all namespaces. -func newTLSRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources) error { +func newTLSRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources, referenceStore *providerReferenceStore) error { r := &tlsRouteReconciler{ client: mgr.GetClient(), log: cfg.Logger, classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), statusUpdater: su, resources: resources, + referenceStore: referenceStore, } c, err := controller.New("tlsroute", mgr, controller.Options{Reconciler: r}) @@ -238,6 +242,10 @@ func (r *tlsRouteReconciler) Reconcile(ctx context.Context, request reconcile.Re // the resource map if it exists. if _, ok := r.resources.Services.Load(svcKey); ok { r.resources.Services.Delete(svcKey) + r.referenceStore.removeRouteToServicesMapping( + ObjectKindNamespacedName{kindTLSRoute, route.Namespace, route.Name}, + svcKey, + ) log.Info("deleted service from resource map") } } @@ -247,6 +255,10 @@ func (r *tlsRouteReconciler) Reconcile(ctx context.Context, request reconcile.Re // The backendRef Service exists, so add it to the resource map. r.resources.Services.Store(svcKey, svc) + r.referenceStore.updateRouteToServicesMapping( + ObjectKindNamespacedName{kindTLSRoute, route.Namespace, route.Name}, + svcKey, + ) log.Info("added service to resource map") } } @@ -257,17 +269,24 @@ func (r *tlsRouteReconciler) Reconcile(ctx context.Context, request reconcile.Re r.resources.TLSRoutes.Delete(request.NamespacedName) log.Info("deleted tlsroute from resource map") - // Delete the Namespace and Service from the resource maps if no other - // routes (TLSRoute or HTTPRoute) exist in the namespace. - found, err := isRoutePresentInNamespace(ctx, r.client, request.NamespacedName.Namespace) - if err != nil { + // Delete the Namespace from the resource maps if no other + // routes (TLSRoute/HTTPRoute) exist in the namespace. + if found, err := isRoutePresentInNamespace(ctx, r.client, request.NamespacedName.Namespace); err != nil { return reconcile.Result{}, err - } - if !found { + } else if !found { r.resources.Namespaces.Delete(request.Namespace) log.Info("deleted namespace from resource map") - r.resources.Services.Delete(request.NamespacedName) - log.Info("deleted service from resource map") + } + + // Delete the Service from the resource maps if no other + // routes (TLSRoute or HTTPRoute) reference that Service. + routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, request.Namespace, request.Name}) + for svc := range routeServices { + r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, request.Namespace, request.Name}, svc) + if r.referenceStore.isServiceReferredByRoutes(svc) { + r.resources.Services.Delete(request.NamespacedName) + log.Info("deleted service from resource map") + } } } From c78103f27480a8fa0308d0fc387ff1f79909aa75 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Wed, 19 Oct 2022 03:39:22 +0800 Subject: [PATCH 043/113] fix: make kube-demo svc mismatched (#597) Signed-off-by: bitliu --- tools/make/kube.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 4025dfbfe76..4ca6c68699e 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -49,8 +49,9 @@ kube-undeploy: manifests $(tools/kustomize) ## Uninstall the Envoy Gateway into .PHONY: kube-demo kube-demo: ## Deploy a demo backend service, gatewayclass, gateway and httproute resource and test the configuration. kubectl apply -f examples/kubernetes/quickstart.yaml + $(eval ENVOY_SERVICE := $(shell kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}')) @echo "\nPort forward to the Envoy service using the command below" - @echo "kubectl -n envoy-gateway-system port-forward service/envoy-default-eg 8888:8080 &" + @echo 'kubectl -n envoy-gateway-system port-forward service/$(ENVOY_SERVICE) 8888:8080 &' @echo "\nCurl the app through Envoy proxy using the command below" @echo "curl --verbose --header \"Host: www.example.com\" http://localhost:8888/get\n" From 84bf298a187a37653e2985923230b5d038f9538b Mon Sep 17 00:00:00 2001 From: Alice Wasko Date: Tue, 18 Oct 2022 16:55:21 -0700 Subject: [PATCH 044/113] Add HTTPRoute request header docs (#585) add httproute request header user doc Signed-off-by: AliceProxy Signed-off-by: AliceProxy --- docs/user/HTTP_REQUEST_HEADERS.md | 308 ++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 docs/user/HTTP_REQUEST_HEADERS.md diff --git a/docs/user/HTTP_REQUEST_HEADERS.md b/docs/user/HTTP_REQUEST_HEADERS.md new file mode 100644 index 00000000000..dd31a1f6d1e --- /dev/null +++ b/docs/user/HTTP_REQUEST_HEADERS.md @@ -0,0 +1,308 @@ +# HTTP Request Headers + +The [HTTPRoute][] resource can modify the headers of a request before forwarding it to the upstream service. HTTPRoute rules cannot use both filter types at once. Currently, Envoy Gateway only supports __core__ +[HTTPRoute filters][] which consist of `RequestRedirect` and `RequestHeaderModifier` at the time of this writing. To +learn more about HTTP routing, refer to the [Gateway API documentation][]. + +A [`RequestHeaderModifier` filter][req_filter] instructs Gateways to modify the headers in requests that match the rule before forwarding the request upstream. Note that the `RequestHeaderModifier` filter will only modify headers before the request is sent from Envoy to the upstream service and will not affect response headers returned to the downstream client. + +Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and then install the example resources used for this guide. + +## Adding Request Headers + +The `RequestHeaderModifier` filter can add new headers to a request before it is sent to the upstream. If the request does not have the header configured by the filter, then that header will be added to the request. If the request already has the header configured by the filter, then the value of the header in the filter will be appended to the value of the header in the request. + +```shell +cat < GET /get HTTP/1.1 +> Host: headers.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< +... + "headers": { + "Accept": [ + "*/*" + ], + "Add-Header": [ + "something", + "foo" + ], +... +``` + +## Setting Request Headers + +Setting headers is similar to adding headers. If the request does not have the header configured by the filter, then it will be added, but unlike [adding request headers](#adding-request-headers) which will append the value of the header if the request already contains it, setting a header will cause the value to be replaced by the value configured in the filter. + +```shell +cat < GET /get HTTP/1.1 +> Host: headers.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< + "headers": { + "Accept": [ + "*/*" + ], + "Set-Header": [ + "foo" + ], +... +``` + +## Removing Request Headers + +Headers can be removed from a request by simply supplying a list of header names. + +Setting headers is similar to adding headers. If the request does not have the header configured by the filter, then it will be added, but unlike [adding request headers](#adding-request-headers) which will append the value of the header if the request already contains it, setting a header will cause the value to be replaced by the value configured in the filter. + +```shell +cat < GET /get HTTP/1.1 +> Host: headers.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< + + "headers": { + "Accept": [ + "*/*" + ], + "Add-Header": [ + "something" + ], +... +``` + +## Combining Filters + +Headers can be added/set/removed in separate filters on the same HTTPRoute and they will all perform as expected + +```shell +cat < Date: Tue, 18 Oct 2022 19:55:03 -0700 Subject: [PATCH 045/113] Add HTTPRoute traffic split doc (#586) add traffic split user doc Signed-off-by: AliceProxy Signed-off-by: AliceProxy --- docs/user/HTTP_TRAFFIC_SPLITTING.md | 294 ++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 docs/user/HTTP_TRAFFIC_SPLITTING.md diff --git a/docs/user/HTTP_TRAFFIC_SPLITTING.md b/docs/user/HTTP_TRAFFIC_SPLITTING.md new file mode 100644 index 00000000000..04ec348514b --- /dev/null +++ b/docs/user/HTTP_TRAFFIC_SPLITTING.md @@ -0,0 +1,294 @@ +# HTTPRoute Traffic Splitting + +The [HTTPRoute][] resource allows one or more [backendRefs][] to be provided. Requests will be routed to these upstreams if they match the rules of the HTTPRoute. If an invalid backendRef is configured, then HTTP responses will be returned with status code `500` for all of the requests that would have been sent to that backend. + +Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and then install the example resources used for this guide. + +## Single backendRef + +When a single backendRef is configured in a HTTPRoute, it will receive 100% of the traffic. + +```shell +cat < GET /get HTTP/1.1 +> Host: backends.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< +... + "namespace": "default", + "ingress": "", + "service": "", + "pod": "backend-79665566f5-s589f" +... +``` + +## Multiple backendRefs + +If multiple backendRefs are configured, then traffic will be split between the backendRefs equally unless a weight is configured. + +First, create a second instance of the example app from the quickstart: + +```shell +cat < GET /get HTTP/1.1 +> Host: backends.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< +... + "namespace": "default", + "ingress": "", + "service": "", + "pod": "backend-75bcd4c969-lsxpz" +... +``` + +## Weighted backendRefs + +If multiple backendRefs are configured and an un-even traffic split between the backends is desired, then the `weight` field can be used to control the weight of requests to each backend. If weight is not configured for a backendRef it is assumed to be `1`. + +The [weight field in a backendRef][backendRefs] controls the distribution of the traffic split. The proportion of requests to a single backendRef is calculated by dividing its `weight` by the sum of all backendRef weights in the HTTPRoute. The weight is not a percentage and the sum of all weights does not need to add up to 100. + +The HTTPRoute below will configure the gateway to send 80% of the traffic to the backend service, and 20% to the backend-2 service. + +```shell +cat < GET /get HTTP/1.1 +> Host: backends.example +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 500 Internal Server Error +< server: envoy +< content-length: 0 +< +``` + +[HTTPRoute]: https://gateway-api.sigs.k8s.io/api-types/httproute/ +[backendRefs]: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.BackendRef From 508af68b9fa98bba02652a4652f851d1dff89bd6 Mon Sep 17 00:00:00 2001 From: Shubham Chauhan Date: Wed, 19 Oct 2022 08:30:37 +0530 Subject: [PATCH 046/113] tls-passthrough docs and quickstart additions (#575) * tls-passthrough docs and quickstart additions Signed-off-by: Shubham Chauhan * minor port fix Signed-off-by: Shubham Chauhan * lintfix Signed-off-by: Shubham Chauhan * review comments Signed-off-by: Shubham Chauhan * lintfix Signed-off-by: Shubham Chauhan * lintfix Signed-off-by: Shubham Chauhan * minor Signed-off-by: Shubham Chauhan * review comments Signed-off-by: Shubham Chauhan Signed-off-by: Shubham Chauhan --- docs/user/tls-passthrough.md | 113 +++++++++++++++++++++++ examples/kubernetes/tls-passthrough.yaml | 73 +++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 docs/user/tls-passthrough.md create mode 100644 examples/kubernetes/tls-passthrough.yaml diff --git a/docs/user/tls-passthrough.md b/docs/user/tls-passthrough.md new file mode 100644 index 00000000000..4416bdb2a4c --- /dev/null +++ b/docs/user/tls-passthrough.md @@ -0,0 +1,113 @@ +# TLS Passthrough +This guide will walk through the steps required to configure TLS Passthrough via Envoy Gateway. Unlike configuring Secure Gateways, where the Gateway terminates the client TLS connection, TLS Passthrough allows the application itself to terminate the TLS connection, while the Gateway routes the requests to the application based on SNI headers. + + +## Prerequisites +- A Kubernetes cluster with `kubectl` context configured for the cluster. +- OpenSSL to generate TLS assets. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. + +## Installation +Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and the example manifest. +Before proceeding, you should be able to curl the example backend using HTTP. + +## TLS Certificates + +Generate the certificates and keys used by the Service to terminate client TLS connections. +For the application, we'll deploy a sample echoserver app, with the certificates loaded in the application Pod. + +__Note:__ These certificates will not be used by the Gateway, but will remain in the application scope. + +Create a root certificate and private key to sign certificates: + +```shell +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=example Inc./CN=example.com' -keyout example.com.key -out example.com.crt +``` + +Create a certificate and a private key for `passthrough.example.com`: + +```shell +openssl req -out passthrough.example.com.csr -newkey rsa:2048 -nodes -keyout passthrough.example.com.key -subj "/CN=passthrough.example.com/O=some organization" +openssl x509 -req -sha256 -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in passthrough.example.com.csr -out passthrough.example.com.crt +``` + +Store the cert/keys in A Secret: + +```shell +kubectl create secret tls server-certs --key=passthrough.example.com.key --cert=passthrough.example.com.crt +``` + +## Deployment +Deploy TLS Passthrough application Deployment, Service and TLSRoute: + +```shell +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/tls-passthrough.yaml +``` + +Patch the Gateway from the Quickstart guide to include a TLS listener that listens on port `6443` and is configured for TLS mode Passthrough: + +```console +$ kubectl patch gateway eg --type=json --patch '[{ + "op": "add", + "path": "/spec/listeners/-", + "value": { + "name": "tls", + "protocol": "TLS", + "hostname": "passthrough.example.com", + "tls": {"mode": "Passthrough"}, + "port": 6443, + }, +}]' +``` + +## Testing + +### Clusters without External LoadBalancer Support + +Get the name of the Envoy service created the by the example Gateway: + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 6043:6443 & +``` + +Curl the example app through Envoy proxy: + +```shell +curl -v --resolve "passthrough.example.com:6043:127.0.0.1" https://passthrough.example.com:6043 \ +--cacert passthrough.example.com.crt +``` + +### Clusters with External LoadBalancer Support +You can also test the same functionality by sending traffic to the External IP of the Gateway: + +```shell +export GATEWAY_HOST=$(kubectl get gateway/eg -o jsonpath='{.status.addresses[0].value}') +``` + +Curl the example app through the Gateway, e.g. Envoy proxy: + +```shell +curl -v -HHost:passthrough.example.com --resolve "passthrough.example.com:6443:${GATEWAY_HOST}" \ +--cacert example.com.crt https://passthrough.example.com:6443/get +``` + +## Clean-Up +Follow the steps from the [Quickstart Guide](QUICKSTART.md) to uninstall Envoy Gateway and the example manifest. + +Delete the Secret: + +```shell +kubectl delete secret/server-certs +``` + +## Next Steps +Checkout the [Developer Guide](../../DEVELOPER.md) to get involved in the project. + +[kind]: https://kind.sigs.k8s.io/ diff --git a/examples/kubernetes/tls-passthrough.yaml b/examples/kubernetes/tls-passthrough.yaml new file mode 100644 index 00000000000..e079132150c --- /dev/null +++ b/examples/kubernetes/tls-passthrough.yaml @@ -0,0 +1,73 @@ +apiVersion: v1 +kind: Service +metadata: + name: passthrough-echoserver + labels: + run: passthrough-echoserver +spec: + ports: + - port: 443 + targetPort: 8443 + protocol: TCP + selector: + run: passthrough-echoserver +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: passthrough-echoserver +spec: + selector: + matchLabels: + run: passthrough-echoserver + replicas: 1 + template: + metadata: + labels: + run: passthrough-echoserver + spec: + containers: + - name: passthrough-echoserver + image: gcr.io/k8s-staging-ingressconformance/echoserver:v20220815-e21d1a4 + ports: + - containerPort: 8443 + env: + - name: HTTPS_PORT + value: "8443" + - name: TLS_SERVER_CERT + value: /etc/server-certs/tls.crt + - name: TLS_SERVER_PRIVKEY + value: /etc/server-certs/tls.key + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: server-certs + mountPath: /etc/server-certs + readOnly: true + volumes: + - name: server-certs + secret: + secretName: server-certs +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TLSRoute +metadata: + name: tlsroute +spec: + parentRefs: + - name: eg + hostnames: + - "passthrough.example.com" + rules: + - backendRefs: + - group: "" + kind: Service + name: passthrough-echoserver + port: 443 + weight: 1 From 4d722b22593daa5e735751e29670804e8606af66 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Wed, 19 Oct 2022 11:34:10 +0800 Subject: [PATCH 047/113] docs: update quickstart to use latest release (#595) * docs: update quickstart to use latest release Signed-off-by: bitliu * remove Signed-off-by: bitliu * update Signed-off-by: bitliu Signed-off-by: bitliu --- README.md | 4 +- docs/design/{ROADMAP.md => roadmap.md} | 0 docs/dev/README.md | 2 +- docs/dev/releasing.md | 2 +- docs/index.rst | 7 +- docs/intro/index.rst | 4 +- docs/user/QUICKSTART.md | 129 ------------------ .../user/{HTTP_ROUTING.md => http-routing.md} | 2 +- docs/user/quickstart.md | 81 +++++++++++ 9 files changed, 92 insertions(+), 139 deletions(-) rename docs/design/{ROADMAP.md => roadmap.md} (100%) delete mode 100644 docs/user/QUICKSTART.md rename docs/user/{HTTP_ROUTING.md => http-routing.md} (98%) create mode 100644 docs/user/quickstart.md diff --git a/README.md b/README.md index a2d293ab0f5..b48388dc791 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Kubernetes-based application gateway. * [Blog][blog] introducing Envoy Gateway. * [Goals](GOALS.md) -* [Quickstart](./docs/user/QUICKSTART.md) to use Envoy Gateway in a few simple steps. -* [Roadmap](./docs/design/ROADMAP.md) +* [Quickstart](./docs/user/quickstart.md) to use Envoy Gateway in a few simple steps. +* [Roadmap](./docs/design/roadmap.md) ## Contact diff --git a/docs/design/ROADMAP.md b/docs/design/roadmap.md similarity index 100% rename from docs/design/ROADMAP.md rename to docs/design/roadmap.md diff --git a/docs/dev/README.md b/docs/dev/README.md index 8cc7c23f281..2d4e6479cc7 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -116,7 +116,7 @@ Now you are able to view the running Envoy configuration by navigating to `127.0 There are many other endpoints on the [Envoy admin interface][] that may be helpful when debugging. -[Quickstart]: https://github.com/envoyproxy/gateway/blob/main/docs/user/QUICKSTART.md +[Quickstart]: https://github.com/envoyproxy/gateway/blob/main/docs/user/quickstart.md [make]: https://www.gnu.org/software/make/ [Github Actions]: https://docs.github.com/en/actions [workflows]: .github/workflows diff --git a/docs/dev/releasing.md b/docs/dev/releasing.md index 5acb2c210a9..e18d7347b95 100644 --- a/docs/dev/releasing.md +++ b/docs/dev/releasing.md @@ -101,7 +101,7 @@ It's important that the world knows about the release. Use the following steps t [release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes [PR 481]: https://github.com/envoyproxy/gateway/pull/481 [Pull Request]: https://github.com/envoyproxy/gateway/pulls -[Quickstart Guide]: https://github.com/envoyproxy/gateway/blob/main/docs/user/QUICKSTART.md +[Quickstart Guide]: https://github.com/envoyproxy/gateway/blob/main/docs/user/quickstart.md [release GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/release.yaml [release workflow]: https://github.com/envoyproxy/gateway/actions/workflows/release.yaml [image]: https://hub.docker.com/r/envoyproxy/gateway/tags diff --git a/docs/index.rst b/docs/index.rst index ec7467ae4c1..e7711e03d56 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -Envoy Gateway: envoyproxy + Gateway API +Envoy Gateway ======================================= Release |version| (Envoy |envoyVersion|, Gateway API |gatewayAPIVersion|) @@ -20,9 +20,10 @@ standalone or Kubernetes-based application gateway. intro/index intro/compatibility - user/QUICKSTART + user/quickstart + user/http-routing design/system-design - design/ROADMAP + design/roadmap design/gatewayapi-translator design/watching design/config-api diff --git a/docs/intro/index.rst b/docs/intro/index.rst index 7291480d7d9..ef349d9d987 100644 --- a/docs/intro/index.rst +++ b/docs/intro/index.rst @@ -2,8 +2,8 @@ Introduction ============ Envoy Gateway is an open source project for managing Envoy Proxy as a -standalone or Kubernetes-based application gateway. It uses the Gateway -API as its sole configuration language. +standalone or Kubernetes-based application gateway. Currently, it uses +Gateway API as its sole configuration language. Many things are in the scope of Envoy Gateway. Many things are not. Many things (like support for non-Kubernetes instances) will be in scope later, diff --git a/docs/user/QUICKSTART.md b/docs/user/QUICKSTART.md deleted file mode 100644 index c826f410e0b..00000000000 --- a/docs/user/QUICKSTART.md +++ /dev/null @@ -1,129 +0,0 @@ -# Quickstart - -This guide will help you get started with Envoy Gateway in a few simple steps. - -## Prerequisites - -A Kubernetes cluster. - -__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. - -## Installation - -Install the Gateway API CRDs: - -```shell -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/gatewayapi-crds.yaml -``` - -Run Envoy Gateway: - -```shell -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/install.yaml -``` - -Run the example app: - -```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/httpbin.yaml -``` - -The Gateway API resources must be created in the following order. First, create the GatewayClass: - -```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/gatewayclass.yaml -``` - -Create the Gateway: - -```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/gateway.yaml -``` - -Create the HTTPRoute to route traffic through Envoy proxy to the example app: - -```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/httproute.yaml -``` - -### Testing the configuration - -Port forward to the Envoy service: - -```shell -kubectl -n envoy-gateway-system port-forward service/envoy-default-eg 8888:8080 & -``` - -Curl the example app through Envoy proxy: - -```shell -curl --verbose --header "Host: www.example.com" http://localhost:8888/get -``` - -You can replace `get` with any of the supported [httpbin methods][httpbin_methods]. - -### For clusters with External Loadbalancer support - -You can also test the same functionality by sending traffic to the External IP. To get the external IP of the -Envoy service, run: - -```shell -export GATEWAY_HOST=$(kubectl get svc/envoy-default-eg -n envoy-gateway-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -``` - -In certain environments, the load balancer may be exposed using a hostname, instead of an IP address. If so, replace -`ip` in the above command with `hostname`. - -Curl the example app through Envoy proxy: - -```shell -curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST:8080/get -``` - -You can replace `get` with any of the supported [httpbin methods][httpbin_methods]. - -## Clean-Up - -Use the steps in this section to uninstall everything from the quickstart guide. - -Delete the HTTPRoute: - -```shell -kubectl delete httproute/httpbin -``` - -Delete the Gateway: - -```shell -kubectl delete gateway/eg -``` - -Delete the GatewayClass: - -```shell -kubectl delete gc/eg -``` - -Uninstall the example app: - -```shell -kubectl delete -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/httpbin.yaml -``` - -Uninstall Envoy Gateway: - -```shell -kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/install.yaml -``` - -Uninstall Gateway API CRDs: - -```shell -kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0-rc2/gatewayapi-crds.yaml -``` - -## Next Steps - -Checkout the [Developer Guide](../dev/README.md) to get involved in the project. - -[httpbin_methods]: https://httpbin.org/#/HTTP_Methods diff --git a/docs/user/HTTP_ROUTING.md b/docs/user/http-routing.md similarity index 98% rename from docs/user/HTTP_ROUTING.md rename to docs/user/http-routing.md index 3d19062abfd..cc7d445ca2e 100644 --- a/docs/user/HTTP_ROUTING.md +++ b/docs/user/http-routing.md @@ -5,7 +5,7 @@ Kubernetes backends. Currently, the only supported backend supported by Envoy Ga shows how to route traffic based on host, header, and path fields and forward the traffic to different Kubernetes Services. To learn more about HTTP routing, refer to the [Gateway API documentation][]. -Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and then install the example +Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and then install the example resources used for this guide. ```shell diff --git a/docs/user/quickstart.md b/docs/user/quickstart.md new file mode 100644 index 00000000000..993003e2419 --- /dev/null +++ b/docs/user/quickstart.md @@ -0,0 +1,81 @@ +# Quickstart + +This guide will help you get started with Envoy Gateway in a few simple steps. + +## Prerequisites + +A Kubernetes cluster. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. + +## Installation + +Install the Gateway API CRDs and Envoy Gateway: + +```shell +kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/latest/install.yaml +``` + +Install the GatewayClass, Gateway, HTTPRoute and example app: + +```shell +kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/latest/quickstart.yaml +``` + +## Testing the Configuration + +Get the name of the Envoy service created the by the example Gateway: + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8888:8080 & +``` + +Curl the example app through Envoy proxy: + +```shell +curl --verbose --header "Host: www.example.com" http://localhost:8888/get +``` + +### External LoadBalancer Support + +You can also test the same functionality by sending traffic to the External IP. To get the external IP of the +Envoy service, run: + +```shell +export GATEWAY_HOST=$(kubectl get svc/${ENVOY_SERVICE} -n envoy-gateway-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +``` + +In certain environments, the load balancer may be exposed using a hostname, instead of an IP address. If so, replace +`ip` in the above command with `hostname`. + +Curl the example app through Envoy proxy: + +```shell +curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST:8080/get +``` + +## Clean-Up + +Use the steps in this section to uninstall everything from the quickstart guide. + +Delete the GatewayClass, Gateway, HTTPRoute and Example App: + +```shell +kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/latest/quickstart.yaml --ignore-not-found=true +``` + +Delete the Gateway API CRDs and Envoy Gateway: + +```shell +kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/latest/install.yaml --ignore-not-found=true +``` + +## Next Steps + +Checkout the [Developer Guide](../dev/README.md) to get involved in the project. From 8d6bec93f453be88487eb90f2c3ce48914cdf179 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 19 Oct 2022 08:16:52 -0700 Subject: [PATCH 048/113] Adds HTTPRoute Redirect Docs (#565) * Adds HTTPRoute Redirect Docs Signed-off-by: danehans * Renames file and resolves 10-17-22 feedback Signed-off-by: danehans * Adds step to get GATEWAY_HOST Signed-off-by: danehans Signed-off-by: danehans --- docs/user/http-redirect.md | 123 +++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 docs/user/http-redirect.md diff --git a/docs/user/http-redirect.md b/docs/user/http-redirect.md new file mode 100644 index 00000000000..9762c798895 --- /dev/null +++ b/docs/user/http-redirect.md @@ -0,0 +1,123 @@ +# HTTP Redirects + +The [HTTPRoute][] resource can issue redirects to clients or rewrite paths sent upstream using filters. Note that +HTTPRoute rules cannot use both filter types at once. Currently, Envoy Gateway only supports __core__ +[HTTPRoute filters][] which consist of `RequestRedirect` and `RequestHeaderModifier` at the time of this writing. To +learn more about HTTP routing, refer to the [Gateway API documentation][]. + +Follow the steps from the [Secure Gateways](secure-gateways.md) to install Envoy Gateway and the example manifest. Do not +proceed until you can curl the example backend from the Quickstart guide using HTTPS. + +## Redirects +Redirects return HTTP 3XX responses to a client, instructing it to retrieve a different resource. A +[`RequestRedirect` filter][req_filter] instructs Gateways to emit a redirect response to requests that match the rule. +For example, to issue a permanent redirect (301) from HTTP to HTTPS, configure `requestRedirect.statusCode=301` and +`requestRedirect.scheme="https"`: + +```shell +cat < Date: Wed, 19 Oct 2022 09:33:39 -0700 Subject: [PATCH 049/113] Fix Merge for HTTPS listeners (#612) * Fix Merge for HTTPS listeners * findXdsHTTPRouteConfigName should only be executed for HTTP listeners, not for HTTPS since we create a new filterChain for them. Fixes: https://github.com/envoyproxy/gateway/issues/520#issuecomment-1283089969 Signed-off-by: Arko Dasgupta * lint Signed-off-by: Arko Dasgupta Signed-off-by: Arko Dasgupta --- internal/xds/translator/translator.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 810ebe00d64..a1ddc511c5f 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -30,13 +30,11 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { if xdsListener == nil { xdsListener = buildXdsListener(httpListener.Name, httpListener.Address, httpListener.Port) tCtx.AddXdsResource(resource.ListenerType, xdsListener) - } else { + } else if httpListener.TLS == nil { // If an existing listener exists, dont create a new filter chain // for HTTP traffic, match on the Domains field within VirtualHosts // within the same RouteConfiguration instead - if httpListener.TLS == nil { - addFilterChain = false - } + addFilterChain = false // Find the route config associated with this listener that // maps to the filter chain for http traffic // There should only be one of these per xds listener From dfc4c78cddd21500d1b5e82e5b0216e9a9632f30 Mon Sep 17 00:00:00 2001 From: Alex Gervais Date: Wed, 19 Oct 2022 12:36:59 -0400 Subject: [PATCH 050/113] Switching codeowners (#617) Signed-off-by: alex Signed-off-by: alex --- CODEOWNERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS.md b/CODEOWNERS.md index 3051c49994a..023fa11e950 100644 --- a/CODEOWNERS.md +++ b/CODEOWNERS.md @@ -1,3 +1,3 @@ # The following owners, listed in alphabetical order, own everything # in the repo. -* @alexgervais @arkodg @danehans @LukeShu @skriss @youngnick +* @AliceProxy @arkodg @danehans @LukeShu @skriss @youngnick From 30cf06ec56c0af46209bf0d14902f1746e496b73 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 19 Oct 2022 09:39:51 -0700 Subject: [PATCH 051/113] Add a Printable method to Xds IR (#607) * Printable returns a safe copy that can be printed/logged * Reintroduce printing xds ir in the gateway api runner Signed-off-by: Arko Dasgupta Signed-off-by: Arko Dasgupta --- internal/gatewayapi/runner/runner.go | 2 ++ internal/ir/xds.go | 10 ++++++ internal/ir/xds_test.go | 49 ++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index a45ee195ec7..804faf0b799 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -87,6 +87,8 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Translate to IR result := t.Translate(&in) + yamlXdsIR, _ := yaml.Marshal(&result.XdsIR) + r.Logger.WithValues("output", "xds-ir").Info(string(yamlXdsIR)) yamlInfraIR, _ := yaml.Marshal(&result.InfraIR) r.Logger.WithValues("output", "infra-ir").Info(string(yamlInfraIR)) diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 9224762d591..3e98e648f50 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -69,6 +69,16 @@ func (x Xds) GetTCPListener(name string) *TCPListener { return nil } +// Printable returns a deep copy of the resource that can be safely logged. +func (x Xds) Printable() *Xds { + out := x.DeepCopy() + for _, listener := range out.HTTP { + // Omit field + listener.TLS = nil + } + return out +} + // HTTPListener holds the listener configuration. // +k8s:deepcopy-gen=true type HTTPListener struct { diff --git a/internal/ir/xds_test.go b/internal/ir/xds_test.go index 537a01aa0d4..8a6710e2643 100644 --- a/internal/ir/xds_test.go +++ b/internal/ir/xds_test.go @@ -16,6 +16,17 @@ var ( Hostnames: []string{"example.com"}, Routes: []*HTTPRoute{&happyHTTPRoute}, } + happyHTTPSListener = HTTPListener{ + Name: "happy", + Address: "0.0.0.0", + Port: 80, + Hostnames: []string{"example.com"}, + TLS: &TLSListenerConfig{ + ServerCertificate: []byte{1, 2, 3}, + PrivateKey: []byte{1, 2, 3}, + }, + Routes: []*HTTPRoute{&happyHTTPRoute}, + } invalidAddrHTTPListener = HTTPListener{ Name: "invalid-addr", Address: "1.0.0", @@ -638,3 +649,41 @@ func TestValidateStringMatch(t *testing.T) { }) } } + +func TestPrintable(t *testing.T) { + tests := []struct { + name string + input Xds + want *Xds + }{ + { + name: "empty", + input: Xds{}, + want: &Xds{}, + }, + { + name: "http", + input: Xds{ + HTTP: []*HTTPListener{&happyHTTPListener}, + }, + want: &Xds{ + HTTP: []*HTTPListener{&happyHTTPListener}, + }, + }, + { + name: "https", + input: Xds{ + HTTP: []*HTTPListener{&happyHTTPSListener}, + }, + want: &Xds{ + HTTP: []*HTTPListener{&happyHTTPListener}, + }, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, *test.want, *test.input.Printable()) + }) + } +} From 195e5b2cc4197e0c664c35bfa679a00fefb20bee Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 19 Oct 2022 10:38:26 -0700 Subject: [PATCH 052/113] fix deletions for secrets (#611) Fixes: https://github.com/envoyproxy/gateway/issues/588 Signed-off-by: Arko Dasgupta --- internal/provider/kubernetes/gateway.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index 6e19957b6e9..f281353091a 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -290,7 +290,6 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req } found := false - var secrets []corev1.Secret // Set status conditions for all accepted gateways. for i := range acceptedGateways { gw := acceptedGateways[i] @@ -376,9 +375,19 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req } if !found { + gw, ok := r.resources.Gateways.Load(request.NamespacedName) + if !ok { + r.log.Info("failed to find gateway in the watchable map", "namespace", gw.Namespace, "name", gw.Name) + } + r.resources.Gateways.Delete(request.NamespacedName) // Delete the TLS secrets from the resource map if no other managed // Gateways reference them. + secrets, _, err := r.secretsAndRefGrantsForGateway(ctx, gw) + if err != nil { + r.log.Info("failed to get secrets and referencegrants for gateway", + "namespace", gw.Namespace, "name", gw.Name) + } for i := range secrets { secret := secrets[i] referenced, err := r.gatewaysRefSecret(ctx, &secret) @@ -535,6 +544,7 @@ func (r *gatewayReconciler) secretsAndRefGrantsForGateway(ctx context.Context, g } secret := new(corev1.Secret) if err := r.client.Get(ctx, key, secret); err != nil { + r.resources.Secrets.Delete(key) return nil, nil, fmt.Errorf("failed to get secret: %v", err) } secrets = append(secrets, *secret) @@ -548,6 +558,7 @@ func (r *gatewayReconciler) secretsAndRefGrantsForGateway(ctx context.Context, g } secret := new(corev1.Secret) if err := r.client.Get(ctx, key, secret); err != nil { + r.resources.Secrets.Delete(key) return nil, nil, fmt.Errorf("failed to get secret: %v", err) } secrets = append(secrets, *secret) From fc78ec1eb94562b2835ea11cf4f6fac780218723 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 19 Oct 2022 10:39:14 -0700 Subject: [PATCH 053/113] dont add filterChainMatch for wildcard hostnames (#604) * dont add filterChainMatch for wildcard hostnames * skip creating a filterChainMatch with a `*` server_names match since envoy doesnt like it and errors out. Fixes: https://github.com/envoyproxy/gateway/issues/601 Signed-off-by: Arko Dasgupta * rm gateway api check for partial hostnames Signed-off-by: Arko Dasgupta --- ...ute-with-partial-wildcard-hostname.in.yaml | 52 -------------- ...te-with-partial-wildcard-hostname.out.yaml | 68 ------------------- internal/gatewayapi/translator.go | 13 ---- internal/xds/translator/listener.go | 28 +++++--- .../out/xds-ir/simple-tls.listeners.yaml | 9 +-- 5 files changed, 18 insertions(+), 152 deletions(-) delete mode 100644 internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml delete mode 100644 internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml diff --git a/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml b/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml deleted file mode 100644 index ac35ef26721..00000000000 --- a/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.in.yaml +++ /dev/null @@ -1,52 +0,0 @@ -gateways: - - apiVersion: gateway.networking.k8s.io/v1beta1 - kind: Gateway - metadata: - namespace: envoy-gateway - name: gateway-1 - spec: - gatewayClassName: envoy-gateway-class - listeners: - # TODO: add test for partial wildcard - # - name: tls-1 - # protocol: TLS - # hostname: "*w.example.com" - # port: 90 - # tls: - # mode: Passthrough - # allowedRoutes: - # namespaces: - # from: All - - name: tls - protocol: TLS - port: 91 - tls: - mode: Passthrough - allowedRoutes: - namespaces: - from: All -tlsRoutes: - - apiVersion: gateway.networking.k8s.io/v1alpha2 - kind: TLSRoute - metadata: - namespace: default - name: tlsroute-1 - spec: - parentRefs: - - namespace: envoy-gateway - name: gateway-1 - rules: - - backendRefs: - - name: service-1 - namespace: test-service-namespace - port: 8080 -services: - - apiVersion: v1 - kind: Service - metadata: - namespace: default - name: service-1 - spec: - clusterIP: 7.7.7.7 - ports: - - port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml deleted file mode 100644 index feb5f807144..00000000000 --- a/internal/gatewayapi/testdata/tlsroute-with-partial-wildcard-hostname.out.yaml +++ /dev/null @@ -1,68 +0,0 @@ -gateways: - - apiVersion: gateway.networking.k8s.io/v1beta1 - kind: Gateway - metadata: - namespace: envoy-gateway - name: gateway-1 - spec: - gatewayClassName: envoy-gateway-class - listeners: - - name: tls - protocol: TLS - port: 91 - tls: - mode: Passthrough - allowedRoutes: - namespaces: - from: All - status: - listeners: - - name: tls - supportedKinds: - - group: gateway.networking.k8s.io - kind: TLSRoute - attachedRoutes: 0 - conditions: - - type: Ready - status: "False" - reason: Invalid - message: Hostname must not be empty with TLS mode Passthrough. -tlsRoutes: - - apiVersion: gateway.networking.k8s.io/v1alpha2 - kind: TLSRoute - metadata: - namespace: default - name: tlsroute-1 - spec: - parentRefs: - - namespace: envoy-gateway - name: gateway-1 - rules: - - backendRefs: - - name: service-1 - namespace: test-service-namespace - port: 8080 - status: - parents: - - parentRef: - namespace: envoy-gateway - name: gateway-1 - controllerName: gateway.envoyproxy.io/gatewayclass-controller - conditions: - - type: Accepted - status: "False" - reason: NoReadyListeners - message: There are no ready listeners for this parent ref -xdsIR: - envoy-gateway-gateway-1: {} -infraIR: - envoy-gateway-gateway-1: - proxy: - metadata: - labels: - gateway.envoyproxy.io/owning-gateway-name: gateway-1 - gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway - name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest - listeners: - - address: "" diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 6212fd9154e..182d9c10eea 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -494,19 +494,6 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap break } - // With TLS Passthrough, partial wildcards are not allowed in xDS config, so "*", "*w.abc.com" are - // invalid configurations. - // TODO: add regex match to detect partial wildcards like *w.abc.com - if listener.Hostname == nil || *listener.Hostname == "" { - listener.SetCondition( - v1beta1.ListenerConditionReady, - metav1.ConditionFalse, - v1beta1.ListenerReasonInvalid, - "Hostname must not be empty with TLS mode Passthrough.", - ) - break - } - if len(listener.TLS.CertificateRefs) > 0 { listener.SetCondition( v1beta1.ListenerConditionReady, diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index 61bdf557279..95f072b8b0c 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -83,11 +83,7 @@ func addXdsHTTPFilterChain(xdsListener *listener.Listener, irListener *ir.HTTPLi return err } filterChain.TransportSocket = tSocket - filterChain.FilterChainMatch = &listener.FilterChainMatch{ - ServerNames: irListener.Hostnames, - } - - if err := addXdsTLSInspectorFilter(xdsListener); err != nil { + if err := addServerNamesMatch(xdsListener, filterChain, irListener.Hostnames); err != nil { return err } @@ -104,6 +100,21 @@ func addXdsHTTPFilterChain(xdsListener *listener.Listener, irListener *ir.HTTPLi return nil } +func addServerNamesMatch(xdsListener *listener.Listener, filterChain *listener.FilterChain, hostnames []string) error { + // Dont add a filter chain match if the hostname is a wildcard character. + if len(hostnames) > 0 && hostnames[0] != "*" { + filterChain.FilterChainMatch = &listener.FilterChainMatch{ + ServerNames: hostnames, + } + + if err := addXdsTLSInspectorFilter(xdsListener); err != nil { + return err + } + } + + return nil +} + // findXdsHTTPRouteConfigName finds the name of the route config associated with the // http connection manager within the default filter chain. func findXdsHTTPRouteConfigName(xdsListener *listener.Listener) (string, error) { @@ -149,14 +160,9 @@ func addXdsTCPFilterChain(xdsListener *listener.Listener, irListener *ir.TCPList } if irListener.TLS != nil { - filterChain.FilterChainMatch = &listener.FilterChainMatch{ - ServerNames: irListener.TLS.SNIs, - } - - if err := addXdsTLSInspectorFilter(xdsListener); err != nil { + if err := addServerNamesMatch(xdsListener, filterChain, irListener.TLS.SNIs); err != nil { return err } - } xdsListener.FilterChains = append(xdsListener.FilterChains, filterChain) diff --git a/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml index 4e05738e9b5..e98b44a0194 100644 --- a/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml @@ -3,10 +3,7 @@ address: 0.0.0.0 portValue: 10080 filterChains: - - filterChainMatch: - serverNames: - - '*' - filters: + - filters: - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager @@ -42,8 +39,4 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - listenerFilters: - - name: envoy.filters.listener.tls_inspector - typedConfig: - '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector name: first-listener From 52cc557bd1824db98ddaaeea8c6906dd41011d01 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Thu, 20 Oct 2022 02:44:17 +0800 Subject: [PATCH 054/113] fix: panic when removing gateway (#622) * fix: panic when removing gateway Signed-off-by: bitliu --- internal/provider/kubernetes/gateway.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index f281353091a..3a5a668681b 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -377,7 +377,8 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req if !found { gw, ok := r.resources.Gateways.Load(request.NamespacedName) if !ok { - r.log.Info("failed to find gateway in the watchable map", "namespace", gw.Namespace, "name", gw.Name) + r.log.Info("failed to find accepted gateway in the watchable map", "namespace", request.Namespace, "name", request.Name) + return reconcile.Result{}, nil } r.resources.Gateways.Delete(request.NamespacedName) From f4a407a03c5dcb1be9e8578ef8ccfe49df42ebaa Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Thu, 20 Oct 2022 02:50:12 +0800 Subject: [PATCH 055/113] fix: add missing index for posts (#614) * fix: add missing index for posts Signed-off-by: bitliu * update Signed-off-by: bitliu * update Signed-off-by: bitliu * update Signed-off-by: bitliu Signed-off-by: bitliu --- docs/conf.py | 2 +- docs/index.rst | 6 +++++- ...EQUEST_HEADERS.md => http-request-headers.md} | 0 ...IC_SPLITTING.md => http-traffic-splitting.md} | 0 docs/user/tls-passthrough.md | 16 ++++++++++------ 5 files changed, 16 insertions(+), 8 deletions(-) rename docs/user/{HTTP_REQUEST_HEADERS.md => http-request-headers.md} (100%) rename docs/user/{HTTP_TRAFFIC_SPLITTING.md => http-traffic-splitting.md} (100%) diff --git a/docs/conf.py b/docs/conf.py index 6f6d6845e25..8067a182843 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,7 +32,7 @@ project = f'Envoy Gateway {version}' author = 'Envoy Gateway Project Authors' -copyright = '2022 Envoy Gateway Project Authors | ' + fullversion +copyright = 'Envoy Gateway Project Authors | GitHub | ' + fullversion envoyVersion = os.environ["ENVOY_VERSION"] gatewayAPIVersion = os.environ["GATEWAYAPI_VERSION"] diff --git a/docs/index.rst b/docs/index.rst index e7711e03d56..774cfbd653a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ Envoy Gateway Release |version| (Envoy |envoyVersion|, Gateway API |gatewayAPIVersion|) .. image:: https://img.shields.io/badge/slack-join-orange.svg - :target: https://envoyproxy.io/slack + :target: https://envoyproxy.slack.com/archives/C03E6NHLESV :alt: Join the Envoy Slack Envoy Gateway is an open source project for managing Envoy Proxy as a @@ -22,6 +22,10 @@ standalone or Kubernetes-based application gateway. intro/compatibility user/quickstart user/http-routing + user/http-redirect + user/http-traffic-splitting + user/http-request-headers + user/tls-passthrough design/system-design design/roadmap design/gatewayapi-translator diff --git a/docs/user/HTTP_REQUEST_HEADERS.md b/docs/user/http-request-headers.md similarity index 100% rename from docs/user/HTTP_REQUEST_HEADERS.md rename to docs/user/http-request-headers.md diff --git a/docs/user/HTTP_TRAFFIC_SPLITTING.md b/docs/user/http-traffic-splitting.md similarity index 100% rename from docs/user/HTTP_TRAFFIC_SPLITTING.md rename to docs/user/http-traffic-splitting.md diff --git a/docs/user/tls-passthrough.md b/docs/user/tls-passthrough.md index 4416bdb2a4c..abf015006db 100644 --- a/docs/user/tls-passthrough.md +++ b/docs/user/tls-passthrough.md @@ -1,20 +1,22 @@ # TLS Passthrough -This guide will walk through the steps required to configure TLS Passthrough via Envoy Gateway. Unlike configuring Secure Gateways, where the Gateway terminates the client TLS connection, TLS Passthrough allows the application itself to terminate the TLS connection, while the Gateway routes the requests to the application based on SNI headers. +This guide will walk through the steps required to configure TLS Passthrough via Envoy Gateway. Unlike configuring Secure Gateways, where the Gateway terminates the client TLS connection, TLS Passthrough allows the application itself to terminate the TLS connection, while the Gateway routes the requests to the application based on SNI headers. ## Prerequisites + - A Kubernetes cluster with `kubectl` context configured for the cluster. - OpenSSL to generate TLS assets. __Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. ## Installation -Follow the steps from the [Quickstart Guide](QUICKSTART.md) to install Envoy Gateway and the example manifest. + +Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and the example manifest. Before proceeding, you should be able to curl the example backend using HTTP. ## TLS Certificates -Generate the certificates and keys used by the Service to terminate client TLS connections. +Generate the certificates and keys used by the Service to terminate client TLS connections. For the application, we'll deploy a sample echoserver app, with the certificates loaded in the application Pod. __Note:__ These certificates will not be used by the Gateway, but will remain in the application scope. @@ -39,6 +41,7 @@ kubectl create secret tls server-certs --key=passthrough.example.com.key --cert= ``` ## Deployment + Deploy TLS Passthrough application Deployment, Service and TLSRoute: ```shell @@ -85,6 +88,7 @@ curl -v --resolve "passthrough.example.com:6043:127.0.0.1" https://passthrough.e ``` ### Clusters with External LoadBalancer Support + You can also test the same functionality by sending traffic to the External IP of the Gateway: ```shell @@ -99,7 +103,8 @@ curl -v -HHost:passthrough.example.com --resolve "passthrough.example.com:6443:$ ``` ## Clean-Up -Follow the steps from the [Quickstart Guide](QUICKSTART.md) to uninstall Envoy Gateway and the example manifest. + +Follow the steps from the [Quickstart Guide](quickstart.md) to uninstall Envoy Gateway and the example manifest. Delete the Secret: @@ -108,6 +113,5 @@ kubectl delete secret/server-certs ``` ## Next Steps -Checkout the [Developer Guide](../../DEVELOPER.md) to get involved in the project. -[kind]: https://kind.sigs.k8s.io/ +Checkout the [Developer Guide](../dev/README.md) to get involved in the project. From 9b593163826ed5948b3baf81a37821a5f48c35f8 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 19 Oct 2022 11:54:03 -0700 Subject: [PATCH 056/113] Updates Dev Guide for Hashed Naming (#602) Signed-off-by: danehans --- docs/dev/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/dev/README.md b/docs/dev/README.md index 2d4e6479cc7..f9b76efd50e 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -107,10 +107,20 @@ is unspecified, the short SHA of your current branch is used. ### Debugging the Envoy Config -An easy way to view the envoy config that Envoy Gateway is using is to port-forward to the admin interface port (currently `19000`) -on the Envoy deployment that corresponds to a Gateway so that it can be accessed locally. +An easy way to view the envoy config that Envoy Gateway is using is to port-forward to the admin interface port +(currently `19000`) on the Envoy deployment that corresponds to a Gateway so that it can be accessed locally. -`kubectl port-forward deploy/envoy-${GATEWAY_NAMESPACE}-${GATEWAY_NAME} -n envoy-gateway-system 19000:19000` +Get the name of the Envoy deployment. The following example is for Gateway `eg` in the `default` namespace: + +```shell +export ENVOY_DEPLOYMENT=$(kubectl get deploy -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward the admin interface port: + +```shell +kubectl port-forward deploy/envoy-${ENVOY_DEPLOYMENT} -n envoy-gateway-system 19000:19000 +``` Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. From ba27f6446b79460a2cb142a49ba3f82731f5811a Mon Sep 17 00:00:00 2001 From: Shubham Chauhan Date: Thu, 20 Oct 2022 00:32:03 +0530 Subject: [PATCH 057/113] fix service cleanup in resource map add tests (#616) * add test for checking service cleanup in resource map Signed-off-by: Shubham Chauhan --- internal/provider/kubernetes/httproute.go | 6 +- .../provider/kubernetes/kubernetes_test.go | 146 +++++++++++++++++- internal/provider/kubernetes/store_test.go | 2 +- internal/provider/kubernetes/tlsroute.go | 6 +- 4 files changed, 146 insertions(+), 14 deletions(-) diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go index b1ce4cd0e19..2c126a9e6df 100644 --- a/internal/provider/kubernetes/httproute.go +++ b/internal/provider/kubernetes/httproute.go @@ -301,9 +301,9 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.R routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, request.Namespace, request.Name}) for svc := range routeServices { r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, request.Namespace, request.Name}, svc) - if r.referenceStore.isServiceReferredByRoutes(svc) { - r.resources.Services.Delete(request.NamespacedName) - log.Info("deleted service from resource map") + if !r.referenceStore.isServiceReferredByRoutes(svc) { + r.resources.Services.Delete(svc) + log.Info("deleted service from resource map", "namespace", svc.Namespace, "name", svc.Name) } } } diff --git a/internal/provider/kubernetes/kubernetes_test.go b/internal/provider/kubernetes/kubernetes_test.go index 2d6cacbc334..de375f061c4 100644 --- a/internal/provider/kubernetes/kubernetes_test.go +++ b/internal/provider/kubernetes/kubernetes_test.go @@ -59,11 +59,12 @@ func TestProvider(t *testing.T) { }() testcases := map[string]func(context.Context, *testing.T, *Provider, *message.ProviderResources){ - "gatewayclass controller name": testGatewayClassController, - "gatewayclass accepted status": testGatewayClassAcceptedStatus, - "gateway scheduled status": testGatewayScheduledStatus, - "httproute": testHTTPRoute, - "tlsroute": testTLSRoute, + "gatewayclass controller name": testGatewayClassController, + "gatewayclass accepted status": testGatewayClassAcceptedStatus, + "gateway scheduled status": testGatewayScheduledStatus, + "httproute": testHTTPRoute, + "tlsroute": testTLSRoute, + "stale service cleanup route deletion": testServiceCleanupForMultipleRoutes, } for name, tc := range testcases { t.Run(name, func(t *testing.T) { @@ -714,9 +715,7 @@ func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resourc svc := getService("test", ns.Name, map[string]int32{ "tls": 90, }) - require.NoError(t, cli.Create(ctx, svc)) - defer func() { require.NoError(t, cli.Delete(ctx, svc)) }() @@ -794,3 +793,136 @@ func testTLSRoute(ctx context.Context, t *testing.T, provider *Provider, resourc }) } } + +// testServiceCleanupForMultipleRoutes creates multiple Routes pointing to the +// same backend Service, and checks whether the Service is properly removed +// from the resource map after Route deletion. +func testServiceCleanupForMultipleRoutes(ctx context.Context, t *testing.T, provider *Provider, resources *message.ProviderResources) { + cli := provider.manager.GetClient() + + gc := getGatewayClass("service-cleanup-test") + require.NoError(t, cli.Create(ctx, gc)) + defer func() { + require.NoError(t, cli.Delete(ctx, gc)) + }() + + // Create the namespace for the Gateway under test. + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "service-cleanup-test"}} + require.NoError(t, cli.Create(ctx, ns)) + + gw := &gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-cleanup-test", + Namespace: ns.Name, + }, + Spec: gwapiv1b1.GatewaySpec{ + GatewayClassName: gwapiv1b1.ObjectName(gc.Name), + Listeners: []gwapiv1b1.Listener{ + { + Name: "httptest", + Port: gwapiv1b1.PortNumber(int32(8080)), + Protocol: gwapiv1b1.HTTPProtocolType, + }, + { + Name: "tlstest", + Port: gwapiv1b1.PortNumber(int32(8043)), + Protocol: gwapiv1b1.TLSProtocolType, + }, + }, + }, + } + require.NoError(t, cli.Create(ctx, gw)) + defer func() { + require.NoError(t, cli.Delete(ctx, gw)) + }() + + svc := getService("test-common-svc", ns.Name, map[string]int32{ + "http": 80, + "tls": 90, + }) + require.NoError(t, cli.Create(ctx, svc)) + defer func() { + require.NoError(t, cli.Delete(ctx, svc)) + }() + + tlsRoute := gwapiv1a2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tlsroute-test", + Namespace: ns.Name, + }, + Spec: gwapiv1a2.TLSRouteSpec{ + CommonRouteSpec: gwapiv1a2.CommonRouteSpec{ + ParentRefs: []gwapiv1a2.ParentReference{{ + Name: gwapiv1a2.ObjectName(gw.Name), + }}, + }, + Hostnames: []gwapiv1a2.Hostname{"test-tls.hostname.local"}, + Rules: []gwapiv1a2.TLSRouteRule{{ + BackendRefs: []gwapiv1a2.BackendRef{{ + BackendObjectReference: gwapiv1a2.BackendObjectReference{ + Name: "test-common-svc", + }}, + }}, + }, + }, + } + + httpRoute := gwapiv1b1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "httproute-test", + Namespace: ns.Name, + }, + Spec: gwapiv1b1.HTTPRouteSpec{ + CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ + ParentRefs: []gwapiv1b1.ParentReference{{ + Name: gwapiv1b1.ObjectName(gw.Name), + }}, + }, + Hostnames: []gwapiv1b1.Hostname{"test-http.hostname.local"}, + Rules: []gwapiv1b1.HTTPRouteRule{{ + Matches: []gwapiv1b1.HTTPRouteMatch{{ + Path: &gwapiv1b1.HTTPPathMatch{ + Type: gatewayapi.PathMatchTypePtr(gwapiv1b1.PathMatchPathPrefix), + Value: gatewayapi.StringPtr("/"), + }, + }}, + BackendRefs: []gwapiv1b1.HTTPBackendRef{{ + BackendRef: gwapiv1b1.BackendRef{ + BackendObjectReference: gwapiv1b1.BackendObjectReference{ + Name: "test-common-svc", + }, + }, + }}, + }}, + }, + } + + // Create the TLSRoute and HTTPRoute + require.NoError(t, cli.Create(ctx, &tlsRoute)) + require.NoError(t, cli.Create(ctx, &httpRoute)) + + // Check that the Service is present in the resource map + key := types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + } + + require.Eventually(t, func() bool { + rSvc, _ := resources.Services.Load(key) + return rSvc != nil + }, defaultWait, defaultTick) + + // Delete the TLSRoute, and check if the Service is still present + require.NoError(t, cli.Delete(ctx, &tlsRoute)) + require.Eventually(t, func() bool { + rSvc, _ := resources.Services.Load(key) + return rSvc != nil + }, defaultWait, defaultTick) + + // Delete the HTTPRoute, and check if the Service is also removed + require.NoError(t, cli.Delete(ctx, &httpRoute)) + require.Eventually(t, func() bool { + rSvc, _ := resources.Services.Load(key) + return rSvc == nil + }, defaultWait, defaultTick) +} diff --git a/internal/provider/kubernetes/store_test.go b/internal/provider/kubernetes/store_test.go index ca5d17c6ad4..08bb2eb4c6b 100644 --- a/internal/provider/kubernetes/store_test.go +++ b/internal/provider/kubernetes/store_test.go @@ -57,6 +57,6 @@ func testRouteToServicesMappings(t *testing.T, cache *providerReferenceStore) { // Verify that ns1svc2 is still referred by another route (HTTPRoute/ns1/r1) require.Equal(t, true, cache.isServiceReferredByRoutes(ns1svc2)) - // Verify that ns1svc1 is not referred by anu other route + // Verify that ns1svc1 is not referred by any other route require.Equal(t, false, cache.isServiceReferredByRoutes(ns1svc1)) } diff --git a/internal/provider/kubernetes/tlsroute.go b/internal/provider/kubernetes/tlsroute.go index 7aee3b0e39c..b9e43bb478c 100644 --- a/internal/provider/kubernetes/tlsroute.go +++ b/internal/provider/kubernetes/tlsroute.go @@ -283,9 +283,9 @@ func (r *tlsRouteReconciler) Reconcile(ctx context.Context, request reconcile.Re routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, request.Namespace, request.Name}) for svc := range routeServices { r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, request.Namespace, request.Name}, svc) - if r.referenceStore.isServiceReferredByRoutes(svc) { - r.resources.Services.Delete(request.NamespacedName) - log.Info("deleted service from resource map") + if !r.referenceStore.isServiceReferredByRoutes(svc) { + r.resources.Services.Delete(svc) + log.Info("deleted service from resource map", "namespace", svc.Namespace, "name", svc.Name) } } } From b497284e53fab47b5c205e3c8aab8d5919545688 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 19 Oct 2022 13:42:22 -0700 Subject: [PATCH 058/113] fix merging https and http listeners on same port (#621) Gracefully handle the case when an https listener is first created, and then an http one is added later. In such a case the route config within the default filter chain wouldnt exist Signed-off-by: Arko Dasgupta Signed-off-by: Arko Dasgupta --- internal/xds/translator/listener.go | 19 +++++++++---- .../xds-ir/multiple-listeners-same-port.yaml | 20 ++++++------- ...ultiple-listeners-same-port.listeners.yaml | 10 +++---- .../multiple-listeners-same-port.routes.yaml | 12 ++++---- .../multiple-listeners-same-port.secrets.yaml | 4 +-- internal/xds/translator/translator.go | 28 +++++++++---------- 6 files changed, 51 insertions(+), 42 deletions(-) diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index 95f072b8b0c..09fe0cd4d31 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -116,18 +116,27 @@ func addServerNamesMatch(xdsListener *listener.Listener, filterChain *listener.F } // findXdsHTTPRouteConfigName finds the name of the route config associated with the -// http connection manager within the default filter chain. -func findXdsHTTPRouteConfigName(xdsListener *listener.Listener) (string, error) { +// http connection manager within the default filter chain and returns an empty string if +// not found. +func findXdsHTTPRouteConfigName(xdsListener *listener.Listener) string { + if xdsListener == nil || xdsListener.DefaultFilterChain == nil || xdsListener.DefaultFilterChain.Filters == nil { + return "" + } + for _, filter := range xdsListener.DefaultFilterChain.Filters { if filter.Name == wellknown.HTTPConnectionManager { m := new(hcm.HttpConnectionManager) if err := filter.GetTypedConfig().UnmarshalTo(m); err != nil { - return "", err + return "" + } + rds := m.GetRds() + if rds == nil { + return "" } - return m.GetRds().GetRouteConfigName(), nil + return rds.GetRouteConfigName() } } - return "", errors.New("unable to find route config") + return "" } func addXdsTCPFilterChain(xdsListener *listener.Listener, irListener *ir.TCPListener, clusterName string) error { diff --git a/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml b/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml index 431eb838f16..38949ebcbd4 100644 --- a/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/multiple-listeners-same-port.yaml @@ -3,7 +3,10 @@ http: address: "0.0.0.0" port: 10080 hostnames: - - "example.com" + - "foo.com" + tls: + serverCertificate: [99, 101, 114, 116, 45, 100, 97, 116, 97] # byte slice representation of "cert-data" + privateKey: [107, 101, 121, 45, 100, 97, 116, 97] # byte slice representation of "key-data" routes: - name: "first-route" destinations: @@ -13,7 +16,10 @@ http: address: "0.0.0.0" port: 10080 hostnames: - - "example.net" + - "foo.net" + tls: + serverCertificate: [99, 101, 114, 116, 45, 100, 97, 116, 97] # byte slice representation of "cert-data" + privateKey: [107, 101, 121, 45, 100, 97, 116, 97] # byte slice representation of "key-data" routes: - name: "second-route" destinations: @@ -23,10 +29,7 @@ http: address: "0.0.0.0" port: 10080 hostnames: - - "foo.com" - tls: - serverCertificate: [99, 101, 114, 116, 45, 100, 97, 116, 97] # byte slice representation of "cert-data" - privateKey: [107, 101, 121, 45, 100, 97, 116, 97] # byte slice representation of "key-data" + - "example.com" routes: - name: "third-route" destinations: @@ -36,10 +39,7 @@ http: address: "0.0.0.0" port: 10080 hostnames: - - "foo.net" - tls: - serverCertificate: [99, 101, 114, 116, 45, 100, 97, 116, 97] # byte slice representation of "cert-data" - privateKey: [107, 101, 121, 45, 100, 97, 116, 97] # byte slice representation of "key-data" + - "example.net" routes: - name: "fourth-route" destinations: diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml index 5732ab6b533..445e851caa2 100644 --- a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml @@ -21,7 +21,7 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: first-listener + routeConfigName: third-listener statPrefix: http filterChains: - filterChainMatch: @@ -45,7 +45,7 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: third-listener + routeConfigName: first-listener statPrefix: https transportSocket: name: envoy.transport_sockets.tls @@ -53,7 +53,7 @@ '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext commonTlsContext: tlsCertificateSdsSecretConfigs: - - name: third-listener + - name: first-listener sdsConfig: apiConfigSource: apiType: DELTA_GRPC @@ -84,7 +84,7 @@ setNodeOnFirstMessageOnly: true transportApiVersion: V3 resourceApiVersion: V3 - routeConfigName: fourth-listener + routeConfigName: second-listener statPrefix: https transportSocket: name: envoy.transport_sockets.tls @@ -92,7 +92,7 @@ '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext commonTlsContext: tlsCertificateSdsSecretConfigs: - - name: fourth-listener + - name: second-listener sdsConfig: apiConfigSource: apiType: DELTA_GRPC diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml index d5b2375a592..b9793e4baff 100644 --- a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.routes.yaml @@ -1,15 +1,17 @@ - name: first-listener virtualHosts: - domains: - - example.com + - foo.com name: first-listener routes: - match: prefix: / route: cluster: first-route +- name: second-listener + virtualHosts: - domains: - - example.net + - foo.net name: second-listener routes: - match: @@ -19,17 +21,15 @@ - name: third-listener virtualHosts: - domains: - - foo.com + - example.com name: third-listener routes: - match: prefix: / route: cluster: third-route -- name: fourth-listener - virtualHosts: - domains: - - foo.net + - example.net name: fourth-listener routes: - match: diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml index 0464fffd2c7..47ebba7468c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.secrets.yaml @@ -1,10 +1,10 @@ -- name: third-listener +- name: first-listener tlsCertificate: certificateChain: inlineBytes: Y2VydC1kYXRh privateKey: inlineBytes: a2V5LWRhdGE= -- name: fourth-listener +- name: second-listener tlsCertificate: certificateChain: inlineBytes: Y2VydC1kYXRh diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index a1ddc511c5f..630d002e32e 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -31,20 +31,18 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { xdsListener = buildXdsListener(httpListener.Name, httpListener.Address, httpListener.Port) tCtx.AddXdsResource(resource.ListenerType, xdsListener) } else if httpListener.TLS == nil { - // If an existing listener exists, dont create a new filter chain - // for HTTP traffic, match on the Domains field within VirtualHosts - // within the same RouteConfiguration instead - addFilterChain = false // Find the route config associated with this listener that - // maps to the filter chain for http traffic - // There should only be one of these per xds listener - routeName, err := findXdsHTTPRouteConfigName(xdsListener) - if err != nil { - return nil, err - } - xdsRouteCfg = findXdsRouteConfig(tCtx, routeName) - if xdsRouteCfg == nil { - return nil, errors.New("unable to find xds route config") + // maps to the default filter chain for http traffic + routeName := findXdsHTTPRouteConfigName(xdsListener) + if routeName != "" { + // If an existing listener exists, dont create a new filter chain + // for HTTP traffic, match on the Domains field within VirtualHosts + // within the same RouteConfiguration instead + addFilterChain = false + xdsRouteCfg = findXdsRouteConfig(tCtx, routeName) + if xdsRouteCfg == nil { + return nil, errors.New("unable to find xds route config") + } } } @@ -52,11 +50,13 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { if err := addXdsHTTPFilterChain(xdsListener, httpListener); err != nil { return nil, err } + } + // Create a route config if we have not found one yet + if xdsRouteCfg == nil { xdsRouteCfg = &route.RouteConfiguration{ Name: httpListener.Name, } - tCtx.AddXdsResource(resource.RouteType, xdsRouteCfg) } From d548634d5c0f13f8235adc68c3f1ab04696f8a9f Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Thu, 20 Oct 2022 06:02:50 +0800 Subject: [PATCH 059/113] chore: add update-quickstart to update tag in quickstart (#610) Signed-off-by: bitliu Signed-off-by: bitliu --- tools/make/kube.mk | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 4ca6c68699e..574366594c1 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -112,3 +112,10 @@ generate-manifests: $(tools/kustomize) ## Generate Kubernetes release manifests. generate-artifacts: generate-manifests ## Generate release artifacts. cp -r $(ROOT_DIR)/release-notes/$(TAG).yaml $(OUTPUT_DIR)/release-notes.yaml @echo "\033[36m===========> Added: $(OUTPUT_DIR)/release-notes.yaml\033[0m" + +.PHONY: update-quickstart +update-quickstart: ## Update quickstart doc image tags to a specific version. + cp -r docs/user/quickstart.md $(OUTPUT_DIR)/quickstart.md + cat $(OUTPUT_DIR)/quickstart.md | sed "s;latest;$(TAG);g" > $(OUTPUT_DIR)/quickstart-$(TAG).md + mv $(OUTPUT_DIR)/quickstart-$(TAG).md docs/user/quickstart.md + @echo "\033[36m===========> Updated: docs/user/quickstart.md\033[0m" From 3b0b5ad10781789804a08b224dacf735f505e8a8 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Thu, 20 Oct 2022 06:04:31 +0800 Subject: [PATCH 060/113] Update Release Procedure (#620) Signed-off-by: bitliu Signed-off-by: bitliu --- docs/dev/releasing.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/dev/releasing.md b/docs/dev/releasing.md index e18d7347b95..987aa40b04f 100644 --- a/docs/dev/releasing.md +++ b/docs/dev/releasing.md @@ -21,12 +21,15 @@ This document guides maintainers through the process of creating an Envoy Gatewa ``` 5. Push the branch to the Envoy Gateway repo. -6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. See [PR 481][] as - a reference for the required changes. +6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. + + ```shell + make update-quickstart TAG=v0.3.0 + ``` + 7. Sign, commit, and push your changes to your fork. Send a PR to get your changes merged into the release branch. Do not proceed until your PR is merged. -8. Confirm that the [release workflow][] for your PR completed successfully. -9. Tag the head of your release branch with the release tag. For example: +8. Tag the head of your release branch with the release tag. For example: ```shell git tag -a v0.3.0 -m 'Envoy Gateway v0.3.0 Release' @@ -34,20 +37,19 @@ This document guides maintainers through the process of creating an Envoy Gatewa __Note:__ The tag version differs from the release branch by including the `.0` patch version. -10. Push the tag to the Envoy Gateway repository. +9. Push the tag to the Envoy Gateway repository. ```shell - git push --tags + git push v0.3.0 ``` -11. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. -12. Confirm that the [release workflow][] completed successfully. -13. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. -14. Confirm that the [release][] was created. -15. Confirm that the steps in the [Quickstart Guide][] work as expected. -16. [Generate][] the GitHub changelog. -17. Submit a PR to merge the Quickstart Guide changes from the release branch into the main branch. -18. If you find any bugs in this process, please create an issue. +10. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +11. Confirm that the [release workflow][] completed successfully. +12. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +13. Confirm that the [release][] was created. +14. Confirm that the steps in the [Quickstart Guide][] work as expected. +15. [Generate][] the GitHub changelog. +16. If you find any bugs in this process, please create an issue. ## Creating a Release Candidate @@ -66,7 +68,7 @@ This document guides maintainers through the process of creating an Envoy Gatewa 5. Push the tag to the Envoy Gateway repository. ```shell - git push --tags + git push v0.3.0-rc.1 ``` 6. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. @@ -99,7 +101,6 @@ It's important that the world knows about the release. Use the following steps t Include a sentence or two that highlights key aspects of the release. [release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes -[PR 481]: https://github.com/envoyproxy/gateway/pull/481 [Pull Request]: https://github.com/envoyproxy/gateway/pulls [Quickstart Guide]: https://github.com/envoyproxy/gateway/blob/main/docs/user/quickstart.md [release GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/release.yaml From 7f66eb99ed9a7cd63f8bc805ccca6f9588ca869a Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 19 Oct 2022 15:47:44 -0700 Subject: [PATCH 061/113] Adds Xunzhuo as a Maintainer (#618) --- CODEOWNERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS.md b/CODEOWNERS.md index 023fa11e950..4f7b014ddd7 100644 --- a/CODEOWNERS.md +++ b/CODEOWNERS.md @@ -1,3 +1,3 @@ # The following owners, listed in alphabetical order, own everything # in the repo. -* @AliceProxy @arkodg @danehans @LukeShu @skriss @youngnick +* @AliceProxy @arkodg @danehans @LukeShu @skriss @Xunzhuo @youngnick From 31e921ef92897fb49382836969d9c9985fdeddef Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 19 Oct 2022 16:34:49 -0700 Subject: [PATCH 062/113] Rename Canary Release workflow to Latest Release (#626) Signed-off-by: Arko Dasgupta Signed-off-by: Arko Dasgupta --- .../{pre_release.yaml => latest_release.yaml} | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) rename .github/workflows/{pre_release.yaml => latest_release.yaml} (66%) diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/latest_release.yaml similarity index 66% rename from .github/workflows/pre_release.yaml rename to .github/workflows/latest_release.yaml index 9d6b055478a..9488a5e480a 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/latest_release.yaml @@ -1,4 +1,4 @@ -name: Canary Release +name: Latest Release on: push: @@ -8,7 +8,7 @@ on: - "**/*.png" jobs: - canary-release: + latest-release: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -16,9 +16,8 @@ jobs: - name: Generate Release Manifests run: make generate-manifests IMAGE=envoyproxy/gateway-dev TAG=latest OUTPUT_DIR=release-artifacts - # Ignore the error from the first time canary release deletion. - # We do not have the release at first, after that, the error will not appear again. - - name: Delete Canary Release + # Ignore the error when we delete the latest release, it might not exist. + - name: Delete the Latest Release continue-on-error: true run: | gh release delete latest --repo $GITHUB_REPOSITORY @@ -26,9 +25,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository_owner }}/${{ github.event.repository.name }} - # Same as above, ignore the error from the first time canary tag deletion. - # We do not have the tag at first, after that, the error will not appear again. - - name: Delete Canary Tag + # Ignore the error when we delete the latest tag, it might not exist. + - name: Delete the Latest Tag continue-on-error: true run: gh api --method DELETE /repos/$GITHUB_REPOSITORY/git/refs/tags/latest @@ -36,7 +34,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository_owner }}/${{ github.event.repository.name }} - - name: Recreate Canary Release and Tag + - name: Recreate the Latest Release and Tag uses: softprops/action-gh-release@v1 with: draft: false @@ -46,8 +44,8 @@ jobs: release-artifacts/install.yaml release-artifacts/quickstart.yaml body: | - This is a "Canary Release" of **Envoy Gateway**, which contains the most recent commits on our main branch. + This is the "latest" release of **Envoy Gateway**, which contains the most recent commits from the main branch. - Canary is **not stable**. + This release **might not be stable**. It is only intended for developers wishing to try out the latest features in Envoy Gateway, some of which may not be fully implemented. From cba9a5bf62c7f62b3e26a2ecefb398fdbf9db7cd Mon Sep 17 00:00:00 2001 From: Nick Young Date: Thu, 20 Oct 2022 12:10:39 +1100 Subject: [PATCH 063/113] Update licensing (#590) * Add license boilerplate and boilerplate check Signed-off-by: Nick Young * Rebase and add new files Signed-off-by: Nick Young * Poke for CI Signed-off-by: Nick Young * Fix PR comments Signed-off-by: Nick Young * Fix second round of PR comments Signed-off-by: Nick Young * Testing golangci Acion Signed-off-by: Nick Young * Revert to boilerplate.py Signed-off-by: Nick Young Signed-off-by: Nick Young --- .github/workflows/build_and_test.yaml | 7 + .golangci.yml | 46 ++++ api/config/v1alpha1/envoygateway_types.go | 5 + api/config/v1alpha1/envoyproxy_types.go | 5 + api/config/v1alpha1/groupversion_info.go | 5 + api/config/v1alpha1/helpers.go | 5 + api/config/v1alpha1/zz_generated.deepcopy.go | 5 + cmd/envoy-gateway/main.go | 17 +- internal/cmd/certgen.go | 5 + internal/cmd/certgen_test.go | 5 + internal/cmd/root.go | 17 +- internal/cmd/root_test.go | 17 +- internal/cmd/server.go | 5 + internal/cmd/server_test.go | 17 +- internal/cmd/versions.go | 5 + internal/cmd/xdstest.go | 5 + internal/crypto/certgen.go | 5 + internal/crypto/certgen_test.go | 5 + internal/envoygateway/config/config.go | 5 + internal/envoygateway/config/decoder.go | 5 + internal/envoygateway/config/decoder_test.go | 5 + internal/envoygateway/scheme.go | 5 + internal/gatewayapi/contexts.go | 5 + internal/gatewayapi/contexts_test.go | 5 + internal/gatewayapi/helpers.go | 5 + internal/gatewayapi/helpers_v1alpha2.go | 11 +- internal/gatewayapi/runner/runner.go | 5 + internal/gatewayapi/runner/runner_test.go | 5 + internal/gatewayapi/sort.go | 5 + internal/gatewayapi/translator.go | 5 + internal/gatewayapi/translator_test.go | 5 + .../infrastructure/kubernetes/configmap.go | 5 + .../kubernetes/configmap_test.go | 5 + .../infrastructure/kubernetes/deployment.go | 5 + .../kubernetes/deployment_test.go | 5 + internal/infrastructure/kubernetes/infra.go | 5 + .../infrastructure/kubernetes/infra_test.go | 5 + internal/infrastructure/kubernetes/labels.go | 5 + .../infrastructure/kubernetes/labels_test.go | 5 + internal/infrastructure/kubernetes/service.go | 5 + .../infrastructure/kubernetes/service_test.go | 5 + .../kubernetes/serviceaccount.go | 5 + .../kubernetes/serviceaccount_test.go | 5 + internal/infrastructure/manager.go | 5 + internal/infrastructure/runner/runner.go | 5 + internal/ir/infra.go | 5 + internal/ir/infra_test.go | 5 + internal/ir/xds.go | 5 + internal/ir/xds_test.go | 5 + internal/ir/zz_generated.deepcopy.go | 5 + internal/log/log.go | 5 + internal/message/types.go | 5 + internal/message/types_test.go | 5 + internal/message/watchutil.go | 5 + internal/message/watchutil_test.go | 5 + internal/provider/kubernetes/gateway.go | 13 +- internal/provider/kubernetes/gateway_test.go | 5 + internal/provider/kubernetes/gatewayclass.go | 7 +- .../provider/kubernetes/gatewayclass_test.go | 5 + internal/provider/kubernetes/helpers.go | 5 + internal/provider/kubernetes/httproute.go | 12 +- .../provider/kubernetes/httproute_test.go | 5 + internal/provider/kubernetes/kubernetes.go | 5 + .../provider/kubernetes/kubernetes_test.go | 7 + internal/provider/kubernetes/rbac.go | 5 + internal/provider/kubernetes/secrets.go | 5 + internal/provider/kubernetes/store.go | 5 + internal/provider/kubernetes/store_test.go | 5 + internal/provider/kubernetes/tlsroute.go | 12 +- internal/provider/runner/runner.go | 5 + internal/provider/runner/runner_test.go | 5 + internal/provider/utils/utils.go | 5 + internal/status/conditions.go | 12 +- internal/status/conditions_test.go | 12 +- internal/status/gateway.go | 5 + internal/status/gatewayclass.go | 12 +- internal/status/status.go | 12 +- internal/utils/env/env.go | 5 + internal/utils/env/env_test.go | 5 + internal/utils/slice/slice.go | 5 + internal/utils/slice/slice_test.go | 5 + internal/xds/cache/logrwrapper.go | 5 + internal/xds/cache/snapshotcache.go | 12 +- internal/xds/server/runner/runner.go | 5 + internal/xds/server/runner/runner_test.go | 5 + internal/xds/translator/cluster.go | 5 + internal/xds/translator/listener.go | 5 + internal/xds/translator/route.go | 5 + internal/xds/translator/runner/runner.go | 5 + internal/xds/translator/runner/runner_test.go | 5 + internal/xds/translator/translator.go | 5 + internal/xds/translator/translator_test.go | 5 + internal/xds/types/resourceversiontable.go | 5 + .../xds/types/resourceversiontable_test.go | 5 + test/conformance/conformance_test.go | 7 + tools/boilerplate/boilerplate.generatego.txt | 5 + tools/boilerplate/boilerplate.go.txt | 5 + tools/boilerplate/boilerplate.py | 257 ++++++++++++++++++ tools/boilerplate/verify-boilerplate.sh | 38 +++ tools/make/kube.mk | 5 +- tools/make/lint.mk | 4 + tools/src/controller-gen/pin.go | 5 + tools/src/golangci-lint/pin.go | 5 + tools/src/goversion/pin.go | 5 + tools/src/kind/pin.go | 5 + tools/src/kustomize/pin.go | 7 +- tools/src/setup-envtest/pin.go | 5 + 107 files changed, 916 insertions(+), 65 deletions(-) create mode 100644 .golangci.yml create mode 100644 tools/boilerplate/boilerplate.generatego.txt create mode 100644 tools/boilerplate/boilerplate.go.txt create mode 100755 tools/boilerplate/boilerplate.py create mode 100755 tools/boilerplate/verify-boilerplate.sh diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index df574694a14..1cad2db02d6 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -28,6 +28,13 @@ jobs: - uses: ./tools/github-actions/setup-deps - run: make -k gen-check + license-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./tools/github-actions/setup-deps + - run: make -k licensecheck + build-and-test: runs-on: ubuntu-latest needs: [lint, gen-check] diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000000..973f62f91c9 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,46 @@ +run: + deadline: 10m + +linters: + enable: + - bodyclose + - gofmt + - goimports + - revive + - gosec + - misspell + - scopelint + - unconvert + - unparam + - goheader + - gocritic + +linters-settings: + gofmt: + simplify: true + unparam: + check-exported: false + goheader: + # Note that because the format is different (this needs no comment markers), + # updating this text means also updating /tools/boilerplate.txt so that + # `make generate` will update the generated files correctly. + template: |- + Copyright Envoy Gateway Authors + SPDX-License-Identifier: Apache-2.0 + The full text of the Apache license is available in the LICENSE file at + the root of the repo. + +issues: + exclude-rules: + - path: zz_generated + linters: + - goimports + - linters: + - staticcheck + text: "SA1019:" + - path: test/e2e + linters: + - bodyclose + # Show the complete output + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/api/config/v1alpha1/envoygateway_types.go b/api/config/v1alpha1/envoygateway_types.go index f9f742b24b8..08c24edafd5 100644 --- a/api/config/v1alpha1/envoygateway_types.go +++ b/api/config/v1alpha1/envoygateway_types.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package v1alpha1 import ( diff --git a/api/config/v1alpha1/envoyproxy_types.go b/api/config/v1alpha1/envoyproxy_types.go index edaaa2aa30f..2899b7c072b 100644 --- a/api/config/v1alpha1/envoyproxy_types.go +++ b/api/config/v1alpha1/envoyproxy_types.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package v1alpha1 import ( diff --git a/api/config/v1alpha1/groupversion_info.go b/api/config/v1alpha1/groupversion_info.go index 992e6bd3162..9a4070f6231 100644 --- a/api/config/v1alpha1/groupversion_info.go +++ b/api/config/v1alpha1/groupversion_info.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + // Package v1alpha1 contains API Schema definitions for the config v1alpha1 API group. // //+kubebuilder:object:generate=true diff --git a/api/config/v1alpha1/helpers.go b/api/config/v1alpha1/helpers.go index d3578fed74c..6b1fa4c5517 100644 --- a/api/config/v1alpha1/helpers.go +++ b/api/config/v1alpha1/helpers.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package v1alpha1 import ( diff --git a/api/config/v1alpha1/zz_generated.deepcopy.go b/api/config/v1alpha1/zz_generated.deepcopy.go index bc0ebd8c291..96aaf745227 100644 --- a/api/config/v1alpha1/zz_generated.deepcopy.go +++ b/api/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,11 @@ //go:build !ignore_autogenerated // +build !ignore_autogenerated +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + // Code generated by controller-gen. DO NOT EDIT. package v1alpha1 diff --git a/cmd/envoy-gateway/main.go b/cmd/envoy-gateway/main.go index 63a30e0885b..f817151187a 100644 --- a/cmd/envoy-gateway/main.go +++ b/cmd/envoy-gateway/main.go @@ -1,16 +1,7 @@ -// Copyright The Envoy Project Authors - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. package main diff --git a/internal/cmd/certgen.go b/internal/cmd/certgen.go index 70a425a12cb..0db554d6312 100644 --- a/internal/cmd/certgen.go +++ b/internal/cmd/certgen.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package cmd import ( diff --git a/internal/cmd/certgen_test.go b/internal/cmd/certgen_test.go index 009e4590f06..fedd07d2236 100644 --- a/internal/cmd/certgen_test.go +++ b/internal/cmd/certgen_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package cmd import ( diff --git a/internal/cmd/root.go b/internal/cmd/root.go index b1a5e9820a2..ae4c96eb8e4 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -1,16 +1,7 @@ -// Copyright The Envoy Project Authors - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. package cmd diff --git a/internal/cmd/root_test.go b/internal/cmd/root_test.go index 27606c98959..da1c1592270 100644 --- a/internal/cmd/root_test.go +++ b/internal/cmd/root_test.go @@ -1,16 +1,7 @@ -// Copyright The Envoy Project Authors - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. package cmd diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 90e2c28cbd9..87aaf21fa0f 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package cmd import ( diff --git a/internal/cmd/server_test.go b/internal/cmd/server_test.go index b9b5498e466..36b723968a4 100644 --- a/internal/cmd/server_test.go +++ b/internal/cmd/server_test.go @@ -1,16 +1,7 @@ -// Copyright The Envoy Project Authors - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. package cmd diff --git a/internal/cmd/versions.go b/internal/cmd/versions.go index 2439be8836e..b95cbade35c 100644 --- a/internal/cmd/versions.go +++ b/internal/cmd/versions.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package cmd import ( diff --git a/internal/cmd/xdstest.go b/internal/cmd/xdstest.go index 9361c671ea2..86ac81014b9 100644 --- a/internal/cmd/xdstest.go +++ b/internal/cmd/xdstest.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package cmd import ( diff --git a/internal/crypto/certgen.go b/internal/crypto/certgen.go index 406a485552f..cde67ab97ff 100644 --- a/internal/crypto/certgen.go +++ b/internal/crypto/certgen.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package crypto import ( diff --git a/internal/crypto/certgen_test.go b/internal/crypto/certgen_test.go index 01dea266d32..9adbcd752f5 100644 --- a/internal/crypto/certgen_test.go +++ b/internal/crypto/certgen_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package crypto import ( diff --git a/internal/envoygateway/config/config.go b/internal/envoygateway/config/config.go index c8f8562becd..162dcd1c056 100644 --- a/internal/envoygateway/config/config.go +++ b/internal/envoygateway/config/config.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package config import ( diff --git a/internal/envoygateway/config/decoder.go b/internal/envoygateway/config/decoder.go index 2ade5f6fb0d..cb9f0e742d0 100644 --- a/internal/envoygateway/config/decoder.go +++ b/internal/envoygateway/config/decoder.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package config import ( diff --git a/internal/envoygateway/config/decoder_test.go b/internal/envoygateway/config/decoder_test.go index 99a1c481d94..33b30385ad3 100644 --- a/internal/envoygateway/config/decoder_test.go +++ b/internal/envoygateway/config/decoder_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package config import ( diff --git a/internal/envoygateway/scheme.go b/internal/envoygateway/scheme.go index bc7d5775534..085d69eea8b 100644 --- a/internal/envoygateway/scheme.go +++ b/internal/envoygateway/scheme.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package envoygateway import ( diff --git a/internal/gatewayapi/contexts.go b/internal/gatewayapi/contexts.go index cb5d56a2274..c6c00ea590f 100644 --- a/internal/gatewayapi/contexts.go +++ b/internal/gatewayapi/contexts.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package gatewayapi import ( diff --git a/internal/gatewayapi/contexts_test.go b/internal/gatewayapi/contexts_test.go index 5b84fdf379e..b584df2629d 100644 --- a/internal/gatewayapi/contexts_test.go +++ b/internal/gatewayapi/contexts_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package gatewayapi import ( diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index 70464b3e88a..e621c101be8 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package gatewayapi import ( diff --git a/internal/gatewayapi/helpers_v1alpha2.go b/internal/gatewayapi/helpers_v1alpha2.go index c4b2153f2a2..1dc934a27f0 100644 --- a/internal/gatewayapi/helpers_v1alpha2.go +++ b/internal/gatewayapi/helpers_v1alpha2.go @@ -1,4 +1,13 @@ -// Portions of this code are based on code from Contour. +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package gatewayapi diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index 804faf0b799..091b92bf6b5 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/gatewayapi/runner/runner_test.go b/internal/gatewayapi/runner/runner_test.go index 77cdfffa61e..c6a84eb5fa6 100644 --- a/internal/gatewayapi/runner/runner_test.go +++ b/internal/gatewayapi/runner/runner_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/gatewayapi/sort.go b/internal/gatewayapi/sort.go index ecf91bff905..6a7d1cc2a24 100644 --- a/internal/gatewayapi/sort.go +++ b/internal/gatewayapi/sort.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package gatewayapi import ( diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 182d9c10eea..95cc8f23300 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package gatewayapi import ( diff --git a/internal/gatewayapi/translator_test.go b/internal/gatewayapi/translator_test.go index b62664e1e89..a343623537f 100644 --- a/internal/gatewayapi/translator_test.go +++ b/internal/gatewayapi/translator_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package gatewayapi import ( diff --git a/internal/infrastructure/kubernetes/configmap.go b/internal/infrastructure/kubernetes/configmap.go index ce24122cc50..4e8dcfec97d 100644 --- a/internal/infrastructure/kubernetes/configmap.go +++ b/internal/infrastructure/kubernetes/configmap.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/configmap_test.go b/internal/infrastructure/kubernetes/configmap_test.go index a240412cfdd..3c7e4669123 100644 --- a/internal/infrastructure/kubernetes/configmap_test.go +++ b/internal/infrastructure/kubernetes/configmap_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/deployment.go b/internal/infrastructure/kubernetes/deployment.go index d3d19d3253c..f031a41a6dd 100644 --- a/internal/infrastructure/kubernetes/deployment.go +++ b/internal/infrastructure/kubernetes/deployment.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/deployment_test.go b/internal/infrastructure/kubernetes/deployment_test.go index 4fd4a7e1728..764b3ce9b95 100644 --- a/internal/infrastructure/kubernetes/deployment_test.go +++ b/internal/infrastructure/kubernetes/deployment_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index 39e6eb4074b..b52094118b6 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/infra_test.go b/internal/infrastructure/kubernetes/infra_test.go index a13374468bb..26f037a816e 100644 --- a/internal/infrastructure/kubernetes/infra_test.go +++ b/internal/infrastructure/kubernetes/infra_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/labels.go b/internal/infrastructure/kubernetes/labels.go index 3fe9edc2467..263bb9d97d2 100644 --- a/internal/infrastructure/kubernetes/labels.go +++ b/internal/infrastructure/kubernetes/labels.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/labels_test.go b/internal/infrastructure/kubernetes/labels_test.go index 1ff33f8d23a..621efaa872b 100644 --- a/internal/infrastructure/kubernetes/labels_test.go +++ b/internal/infrastructure/kubernetes/labels_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/service.go b/internal/infrastructure/kubernetes/service.go index fd93f81c08b..8441aa978af 100644 --- a/internal/infrastructure/kubernetes/service.go +++ b/internal/infrastructure/kubernetes/service.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/service_test.go b/internal/infrastructure/kubernetes/service_test.go index 679ea8af440..f1ea3d1a2c0 100644 --- a/internal/infrastructure/kubernetes/service_test.go +++ b/internal/infrastructure/kubernetes/service_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/serviceaccount.go b/internal/infrastructure/kubernetes/serviceaccount.go index 3a40f61822d..97110c1d2ca 100644 --- a/internal/infrastructure/kubernetes/serviceaccount.go +++ b/internal/infrastructure/kubernetes/serviceaccount.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/kubernetes/serviceaccount_test.go b/internal/infrastructure/kubernetes/serviceaccount_test.go index d6299d0f128..c02719843ab 100644 --- a/internal/infrastructure/kubernetes/serviceaccount_test.go +++ b/internal/infrastructure/kubernetes/serviceaccount_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index 8ed44db7a34..a263dc1566c 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package infrastructure import ( diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index ce5413b7d80..cb9ec0d551c 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/ir/infra.go b/internal/ir/infra.go index 99d91f103aa..28021eb50bd 100644 --- a/internal/ir/infra.go +++ b/internal/ir/infra.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package ir import ( diff --git a/internal/ir/infra_test.go b/internal/ir/infra_test.go index 3a2f7f28b0a..72524ea37a4 100644 --- a/internal/ir/infra_test.go +++ b/internal/ir/infra_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package ir import ( diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 3e98e648f50..def2bfd5189 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package ir import ( diff --git a/internal/ir/xds_test.go b/internal/ir/xds_test.go index 8a6710e2643..32dd356cd19 100644 --- a/internal/ir/xds_test.go +++ b/internal/ir/xds_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package ir import ( diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 16c3477e5b9..0f26e03b505 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -1,6 +1,11 @@ //go:build !ignore_autogenerated // +build !ignore_autogenerated +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + // Code generated by controller-gen. DO NOT EDIT. package ir diff --git a/internal/log/log.go b/internal/log/log.go index 54371fdb61b..cd51d3a80eb 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package log import ( diff --git a/internal/message/types.go b/internal/message/types.go index 33861192477..e986d576261 100644 --- a/internal/message/types.go +++ b/internal/message/types.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package message import ( diff --git a/internal/message/types_test.go b/internal/message/types_test.go index 4b2faa409bd..9525efac84a 100644 --- a/internal/message/types_test.go +++ b/internal/message/types_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package message import ( diff --git a/internal/message/watchutil.go b/internal/message/watchutil.go index 40f22cb23f2..4fccb9059da 100644 --- a/internal/message/watchutil.go +++ b/internal/message/watchutil.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package message import ( diff --git a/internal/message/watchutil_test.go b/internal/message/watchutil_test.go index abc0ccb5d2d..873c8b2d53c 100644 --- a/internal/message/watchutil_test.go +++ b/internal/message/watchutil_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package message_test import ( diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index 3a5a668681b..f7f958da0a8 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -1,5 +1,14 @@ -// Portions of this code are based on code from Contour, available at: -// https://github.com/projectcontour/contour/blob/main/internal/controller/gateway.go +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file +// https://github.com/projectcontour/contour/blob/main/internal/controller/gateway.go// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package kubernetes diff --git a/internal/provider/kubernetes/gateway_test.go b/internal/provider/kubernetes/gateway_test.go index 8f4248e8cfa..7d7e1fe2530 100644 --- a/internal/provider/kubernetes/gateway_test.go +++ b/internal/provider/kubernetes/gateway_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/gatewayclass.go b/internal/provider/kubernetes/gatewayclass.go index 2f4d0f31897..8a360202bb7 100644 --- a/internal/provider/kubernetes/gatewayclass.go +++ b/internal/provider/kubernetes/gatewayclass.go @@ -1,4 +1,9 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// TODO Portions of this code are based on code from Contour, available at: // https://github.com/projectcontour/contour/blob/main/internal/controller/gatewayclass.go package kubernetes diff --git a/internal/provider/kubernetes/gatewayclass_test.go b/internal/provider/kubernetes/gatewayclass_test.go index a2e8497762e..4200a2379e8 100644 --- a/internal/provider/kubernetes/gatewayclass_test.go +++ b/internal/provider/kubernetes/gatewayclass_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/helpers.go b/internal/provider/kubernetes/helpers.go index 8ca309cecf6..7c94208b203 100644 --- a/internal/provider/kubernetes/helpers.go +++ b/internal/provider/kubernetes/helpers.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go index 2c126a9e6df..e99ef92cbf6 100644 --- a/internal/provider/kubernetes/httproute.go +++ b/internal/provider/kubernetes/httproute.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/controller/httproute.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package kubernetes diff --git a/internal/provider/kubernetes/httproute_test.go b/internal/provider/kubernetes/httproute_test.go index c504d312153..605213984d6 100644 --- a/internal/provider/kubernetes/httproute_test.go +++ b/internal/provider/kubernetes/httproute_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index cc5670a90b6..0a6903e3d3a 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/kubernetes_test.go b/internal/provider/kubernetes/kubernetes_test.go index de375f061c4..13202eed5ef 100644 --- a/internal/provider/kubernetes/kubernetes_test.go +++ b/internal/provider/kubernetes/kubernetes_test.go @@ -1,6 +1,13 @@ //go:build integration // +build integration +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + + + package kubernetes import ( diff --git a/internal/provider/kubernetes/rbac.go b/internal/provider/kubernetes/rbac.go index 4015f405502..8222daec3e5 100644 --- a/internal/provider/kubernetes/rbac.go +++ b/internal/provider/kubernetes/rbac.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes // +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=gatewayclasses;gateways;httproutes;tlsroutes;referencepolicies;referencegrants,verbs=get;list;watch;update diff --git a/internal/provider/kubernetes/secrets.go b/internal/provider/kubernetes/secrets.go index 3027ecb0dc8..b3a0ae251d7 100644 --- a/internal/provider/kubernetes/secrets.go +++ b/internal/provider/kubernetes/secrets.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/store.go b/internal/provider/kubernetes/store.go index 8d9be70e5c4..3ecc514fda5 100644 --- a/internal/provider/kubernetes/store.go +++ b/internal/provider/kubernetes/store.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/store_test.go b/internal/provider/kubernetes/store_test.go index 08bb2eb4c6b..66c3c4bfdf3 100644 --- a/internal/provider/kubernetes/store_test.go +++ b/internal/provider/kubernetes/store_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package kubernetes import ( diff --git a/internal/provider/kubernetes/tlsroute.go b/internal/provider/kubernetes/tlsroute.go index b9e43bb478c..9cd3db39e9c 100644 --- a/internal/provider/kubernetes/tlsroute.go +++ b/internal/provider/kubernetes/tlsroute.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/controller/tlsroute.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package kubernetes diff --git a/internal/provider/runner/runner.go b/internal/provider/runner/runner.go index b8678eb0edd..251acc50e9a 100644 --- a/internal/provider/runner/runner.go +++ b/internal/provider/runner/runner.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/provider/runner/runner_test.go b/internal/provider/runner/runner_test.go index 44bf29c2a68..170df8182e0 100644 --- a/internal/provider/runner/runner_test.go +++ b/internal/provider/runner/runner_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/provider/utils/utils.go b/internal/provider/utils/utils.go index f736356a5ae..353e3ae08d8 100644 --- a/internal/provider/utils/utils.go +++ b/internal/provider/utils/utils.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package utils import ( diff --git a/internal/status/conditions.go b/internal/status/conditions.go index 03dcffcc4a0..f83873fe6c8 100644 --- a/internal/status/conditions.go +++ b/internal/status/conditions.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/status/gatewayclassconditions.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package status diff --git a/internal/status/conditions_test.go b/internal/status/conditions_test.go index 6cc8a52cf99..ba2f2cd21f3 100644 --- a/internal/status/conditions_test.go +++ b/internal/status/conditions_test.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/status/gatewayclassconditions_test.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package status diff --git a/internal/status/gateway.go b/internal/status/gateway.go index feb8ba73182..d802d09e7b8 100644 --- a/internal/status/gateway.go +++ b/internal/status/gateway.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package status import ( diff --git a/internal/status/gatewayclass.go b/internal/status/gatewayclass.go index 41581b22904..2853c258a58 100644 --- a/internal/status/gatewayclass.go +++ b/internal/status/gatewayclass.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/status/gatewayclass.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package status diff --git a/internal/status/status.go b/internal/status/status.go index 373a19b3941..58dca4b576b 100644 --- a/internal/status/status.go +++ b/internal/status/status.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/k8s/status.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package status diff --git a/internal/utils/env/env.go b/internal/utils/env/env.go index 3a91d4b158b..da55be76b83 100644 --- a/internal/utils/env/env.go +++ b/internal/utils/env/env.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package env import ( diff --git a/internal/utils/env/env_test.go b/internal/utils/env/env_test.go index 118d2e8108e..357b69ef4d6 100644 --- a/internal/utils/env/env_test.go +++ b/internal/utils/env/env_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package env import ( diff --git a/internal/utils/slice/slice.go b/internal/utils/slice/slice.go index ad88dfd32da..cc75f9f0e59 100644 --- a/internal/utils/slice/slice.go +++ b/internal/utils/slice/slice.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package slice // ContainsString checks if a given slice of strings contains the provided string. diff --git a/internal/utils/slice/slice_test.go b/internal/utils/slice/slice_test.go index 2ac94bab8c6..8b76632e972 100644 --- a/internal/utils/slice/slice_test.go +++ b/internal/utils/slice/slice_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package slice import ( diff --git a/internal/xds/cache/logrwrapper.go b/internal/xds/cache/logrwrapper.go index 5c1f0732840..a21324e4ba1 100644 --- a/internal/xds/cache/logrwrapper.go +++ b/internal/xds/cache/logrwrapper.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package cache import ( diff --git a/internal/xds/cache/snapshotcache.go b/internal/xds/cache/snapshotcache.go index 82b41e29c90..255bbc591eb 100644 --- a/internal/xds/cache/snapshotcache.go +++ b/internal/xds/cache/snapshotcache.go @@ -1,5 +1,15 @@ -// Portions of this code are based on code from Contour, available at: +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// This file contains code derived from Contour, +// https://github.com/projectcontour/contour +// from the source file // https://github.com/projectcontour/contour/blob/main/internal/xds/v3/snapshotter.go +// and is provided here subject to the following: +// Copyright Project Contour Authors +// SPDX-License-Identifier: Apache-2.0 package cache diff --git a/internal/xds/server/runner/runner.go b/internal/xds/server/runner/runner.go index 38f359b6535..42b65bd44f1 100644 --- a/internal/xds/server/runner/runner.go +++ b/internal/xds/server/runner/runner.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/xds/server/runner/runner_test.go b/internal/xds/server/runner/runner_test.go index bcb62c24429..3d0a86c58de 100644 --- a/internal/xds/server/runner/runner_test.go +++ b/internal/xds/server/runner/runner_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index e63d283eb9d..4f0c479f983 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package translator import ( diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index 09fe0cd4d31..e811f22db1f 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package translator import ( diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index dc5acd70e4b..ac4178e7b49 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package translator import ( diff --git a/internal/xds/translator/runner/runner.go b/internal/xds/translator/runner/runner.go index 515436b708a..49ee70254eb 100644 --- a/internal/xds/translator/runner/runner.go +++ b/internal/xds/translator/runner/runner.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/xds/translator/runner/runner_test.go b/internal/xds/translator/runner/runner_test.go index b6649b1cf74..b7e7d8be032 100644 --- a/internal/xds/translator/runner/runner_test.go +++ b/internal/xds/translator/runner/runner_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package runner import ( diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 630d002e32e..7d6f45629ba 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package translator import ( diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index c968badfa80..ea61d193770 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package translator import ( diff --git a/internal/xds/types/resourceversiontable.go b/internal/xds/types/resourceversiontable.go index 9aed4c728d5..ebf323ab8f1 100644 --- a/internal/xds/types/resourceversiontable.go +++ b/internal/xds/types/resourceversiontable.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package types import ( diff --git a/internal/xds/types/resourceversiontable_test.go b/internal/xds/types/resourceversiontable_test.go index e78402f526b..2cb17be407f 100644 --- a/internal/xds/types/resourceversiontable_test.go +++ b/internal/xds/types/resourceversiontable_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package types import ( diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index b3b748ca43f..8c9730904d8 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -1,6 +1,13 @@ //go:build conformance // +build conformance +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + + + package conformance import ( diff --git a/tools/boilerplate/boilerplate.generatego.txt b/tools/boilerplate/boilerplate.generatego.txt new file mode 100644 index 00000000000..3f9e2deb710 --- /dev/null +++ b/tools/boilerplate/boilerplate.generatego.txt @@ -0,0 +1,5 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + diff --git a/tools/boilerplate/boilerplate.go.txt b/tools/boilerplate/boilerplate.go.txt new file mode 100644 index 00000000000..3f9e2deb710 --- /dev/null +++ b/tools/boilerplate/boilerplate.go.txt @@ -0,0 +1,5 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + diff --git a/tools/boilerplate/boilerplate.py b/tools/boilerplate/boilerplate.py new file mode 100755 index 00000000000..4741440808f --- /dev/null +++ b/tools/boilerplate/boilerplate.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is copied from https://github.com/kubernetes/kubernetes/blob/04c2b1fbdc1289c9a72eda87cf7072346e60d241/hack/boilerplate/boilerplate.py + +from __future__ import print_function + +import argparse +import datetime +import difflib +import glob +import os +import re +import sys + +parser = argparse.ArgumentParser() +parser.add_argument( + "filenames", + help="list of files to check, all files if unspecified", + nargs='*') + +rootdir = os.path.dirname(__file__) + "/../../" +rootdir = os.path.abspath(rootdir) +parser.add_argument( + "--rootdir", default=rootdir, help="root directory to examine") + +default_boilerplate_dir = os.path.join(rootdir, "tools/boilerplate") +parser.add_argument( + "--boilerplate-dir", default=default_boilerplate_dir) + +parser.add_argument( + "-v", "--verbose", + help="give verbose output regarding why a file does not pass", + action="store_true") + +args = parser.parse_args() + +verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") + + +def get_refs(): + refs = {} + + for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + + return refs + + +def is_generated_file(filename, data, regexs): + for d in skipped_ungenerated_files: + if d in filename: + return False + + p = regexs["generated"] + return p.search(data) + + +def file_passes(filename, refs, regexs): + try: + f = open(filename, 'r') + except Exception as exc: + print("Unable to open %s: %s" % (filename, exc), file=verbose_out) + return False + + data = f.read() + f.close() + + # determine if the file is automatically generated + generated = is_generated_file(filename, data, regexs) + + basename = os.path.basename(filename) + extension = file_extension(filename) + if generated: + if extension == "go": + extension = "generatego" + elif extension == "bzl": + extension = "generatebzl" + + if extension != "": + ref = refs[extension] + else: + ref = refs[basename] + + # remove extra content from the top of files + if extension == "go" or extension == "generatego": + p = regexs["go_build_constraints"] + (data, found) = p.subn("", data, 1) + elif extension in ["sh", "py"]: + p = regexs["shebang"] + (data, found) = p.subn("", data, 1) + + data = data.splitlines() + + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + print('File %s smaller than reference (%d < %d)' % + (filename, len(data), len(ref)), + file=verbose_out) + return False + + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + + p = regexs["year"] + for d in data: + if p.search(d): + if generated: + print('File %s has the YEAR field, but it should not be in generated file' % + filename, file=verbose_out) + else: + print('File %s has the YEAR field, but missing the year of date' % + filename, file=verbose_out) + return False + + if not generated: + # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR" + p = regexs["date"] + for i, d in enumerate(data): + (data[i], found) = p.subn('YEAR', d) + if found != 0: + break + + # if we don't match the reference at this point, fail + if ref != data: + print("Header in %s does not match reference, diff:" % + filename, file=verbose_out) + if args.verbose: + print(file=verbose_out) + for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + print(line, file=verbose_out) + print(file=verbose_out) + return False + + return True + + +def file_extension(filename): + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +skipped_dirs = [ + 'cluster/env.sh', + '.git', + '_gopath', + 'hack/boilerplate/test', + '_output', + 'staging/src/k8s.io/kubectl/pkg/generated/bindata.go', + 'test/e2e/generated/bindata.go', + 'third_party', + 'vendor', + '.venv', +] + +# list all the files contain 'DO NOT EDIT', but are not generated +skipped_ungenerated_files = [ + 'hack/lib/swagger.sh', 'tools/boilerplate/boilerplate.py'] + + +def normalize_files(files): + newfiles = [] + for pathname in files: + if any(x in pathname for x in skipped_dirs): + continue + newfiles.append(pathname) + for i, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[i] = os.path.join(args.rootdir, pathname) + return newfiles + + +def get_files(extensions): + files = [] + if len(args.filenames) > 0: + files = args.filenames + else: + for root, dirs, walkfiles in os.walk(args.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for d in skipped_dirs: + if d in dirs: + dirs.remove(d) + + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_dates(): + years = datetime.datetime.now().year + return '(%s)' % '|'.join((str(year) for year in range(2014, years+1))) + + +def get_regexs(): + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + regexs["year"] = re.compile('YEAR') + # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)"; + # company holder names can be anything + regexs["date"] = re.compile(get_dates()) + # strip the following build constraints/tags: + # //go:build + # // +build \n\n + regexs["go_build_constraints"] = re.compile( + r"^(//(go:build| \+build).*\n)+\n", re.MULTILINE) + # strip #!.* from scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + # Search for generated files + regexs["generated"] = re.compile('DO NOT EDIT') + return regexs + + +def main(): + regexs = get_regexs() + refs = get_refs() + filenames = get_files(refs.keys()) + + for filename in filenames: + if not file_passes(filename, refs, regexs): + print(filename, file=sys.stdout) + + print("Verified %d file headers match boilerplate" % (len(filenames),), file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/boilerplate/verify-boilerplate.sh b/tools/boilerplate/verify-boilerplate.sh new file mode 100755 index 00000000000..ab0c7d798f6 --- /dev/null +++ b/tools/boilerplate/verify-boilerplate.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +boilerDir="${SCRIPT_ROOT}/tools/boilerplate" +boiler="${boilerDir}/boilerplate.py" + +files_need_boilerplate=($(${boiler} "$@" -v)) + +echo $SCRIPT_ROOT +echo $boilerDir + +# Run boilerplate check +if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then + for file in "${files_need_boilerplate[@]}"; do + echo "Boilerplate header is wrong for: ${file}" + done + + exit 1 +fi diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 574366594c1..c78929bf09a 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -19,6 +19,8 @@ KUBE_INFRA_DIR := $(ROOT_DIR)/internal/infrastructure/kubernetes/config endif ##@ Kubernetes Development +YEAR := $(shell date +%Y) +CONTROLLERGEN_OBJECT_FLAGS := object:headerFile="$(ROOT_DIR)/tools/boilerplate/boilerplate.generatego.txt",year=$(YEAR) .PHONY: manifests manifests: $(tools/controller-gen) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. @@ -26,7 +28,8 @@ manifests: $(tools/controller-gen) ## Generate WebhookConfiguration, ClusterRole .PHONY: generate generate: $(tools/controller-gen) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(tools/controller-gen) object paths="./..." +# Note that the paths can't just be "./..." with the header file, or the tool will panic on run. Sorry. + $(tools/controller-gen) $(CONTROLLERGEN_OBJECT_FLAGS) paths="{$(ROOT_DIR)/api/config/...,$(ROOT_DIR)/internal/ir/...}" .PHONY: kube-test kube-test: manifests generate $(tools/setup-envtest) ## Run Kubernetes provider tests. diff --git a/tools/make/lint.mk b/tools/make/lint.mk index 9aca38fe94e..465a2181a9b 100644 --- a/tools/make/lint.mk +++ b/tools/make/lint.mk @@ -72,3 +72,7 @@ gen-check: generate manifests echo "\nERROR: Some files need to be updated, please run 'make generate' and 'make manifests' to include any changed files to your PR\n"; \ git diff --exit-code; \ fi + +.PHONY: licensecheck +licensecheck: ## Check license headers are present. + tools/boilerplate/verify-boilerplate.sh diff --git a/tools/src/controller-gen/pin.go b/tools/src/controller-gen/pin.go index 0eb2187c6e5..5140b3786b5 100644 --- a/tools/src/controller-gen/pin.go +++ b/tools/src/controller-gen/pin.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build pin // +build pin diff --git a/tools/src/golangci-lint/pin.go b/tools/src/golangci-lint/pin.go index d250d0323ed..1500aa4c7b9 100644 --- a/tools/src/golangci-lint/pin.go +++ b/tools/src/golangci-lint/pin.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build pin // +build pin diff --git a/tools/src/goversion/pin.go b/tools/src/goversion/pin.go index 662fbb00bd2..83faf7ad75d 100644 --- a/tools/src/goversion/pin.go +++ b/tools/src/goversion/pin.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build pin // +build pin diff --git a/tools/src/kind/pin.go b/tools/src/kind/pin.go index 20964c929c6..5180e168384 100644 --- a/tools/src/kind/pin.go +++ b/tools/src/kind/pin.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build pin // +build pin diff --git a/tools/src/kustomize/pin.go b/tools/src/kustomize/pin.go index 71820da506a..f871f2f9b93 100644 --- a/tools/src/kustomize/pin.go +++ b/tools/src/kustomize/pin.go @@ -1,6 +1,11 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build pin // +build pin package ignore -import "sigs.k8s.io/kustomize/kustomize/v3" +import "sigs.k8s.io/kustomize/kustomize/v3" \ No newline at end of file diff --git a/tools/src/setup-envtest/pin.go b/tools/src/setup-envtest/pin.go index c1aef52f066..69d708e7ea9 100644 --- a/tools/src/setup-envtest/pin.go +++ b/tools/src/setup-envtest/pin.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build pin // +build pin From ac3b50f403ac362eb2e97247fe1963f700b19907 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 19 Oct 2022 18:50:01 -0700 Subject: [PATCH 064/113] Adds v0.2.0 Release Notes (#625) --- release-notes/v0.2.0.yaml | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 release-notes/v0.2.0.yaml diff --git a/release-notes/v0.2.0.yaml b/release-notes/v0.2.0.yaml new file mode 100644 index 00000000000..cecac2f7693 --- /dev/null +++ b/release-notes/v0.2.0.yaml @@ -0,0 +1,52 @@ +date: October 19, 2022 + +changes: + - area: documentation + change: | + Added Config API, translator, roadmap, and message bus design documentation. + Added documentation for releasing Envoy Gateway. + Added user guides for configuring common tasks, e.g. HTTP request routing. + Added support for the Sphinx documentation generator. + - area: api + change: | + Added the EnvoyGateway API type for configuring Envoy Gateway. + Added the EnvoyProxy API type for configuring managed Envoys. + - area: ci-tooling-testing + change: | + Added tooling to build, run, etc. Envoy Gateway. + Added Gateway API conformance tests. + Added Make-based tooling to fetch all tools so checks (code lint, spellchecks) and tests can be run locally. + Added support for releasing latest artifacts to GitHub. + Added code coverage with a minimum 60% threshold. + - area: ir + change: | + Added xds and infra IRs to decouple user-facing APIs from Envoy Gateway. + Added IR validation. + - area: translator + change: | + Added the gatewayapi translator to translate Gateway API and associated resources to the IR and manage the + status of Gateway API resources. + Added the xDS translator to translate the xds IR to xDS resources. + - area: message-service + change: | + Added infra and xds IR watchable map messages for inter-component communication. + Added a Runner to each Envoy Gateway component to support pub/sub between components. + Added support for managing multiple separate Envoy proxy fleets. + - area: infra-manager + change: | + Added Kubernetes Infra Manager to manage Envoy infrastructure running in a Kubernetes cluster. + Added support for managing a separate Envoy infrastructure per Gateway. + - area: providers + change: | + Added the Kubernetes provider with support for managing GatewayClass, Gateway, HTTPRoute, ReferenceGrant, and + TLSRoute resources. + Due to Issue #539, a ReferenceGrant is not removed from the system when unreferenced. + Due to Issue #577, TLSRoute is not being tested for Gateway API conformance. + Added watchers for dependent resources of managed Envoy infrastructure to trigger reconciliation. + Added support for labeling managed infrastructure using Gateway namespace/name labels. + Added support for finalizing the managed GatewayClass. + - area: xds + change: | + Added xDS server support to configure managed Envoys using Delta xDS. + Added initial support for mTLS between the xDS server and managed Envoys. + Due to envoyproxy/go-control-plane Issue #599, Envoy Gateway logs the private key of HTTPS listeners. From 085e58ea672f3bbf82e833d652550179df4238e6 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Fri, 21 Oct 2022 00:31:56 +0800 Subject: [PATCH 065/113] fix: update build-and-test workflow to match the latest branch pattern (#631) * fix: update build-and-test workflow to match the latest branch pattern Signed-off-by: bitliu * update Signed-off-by: bitliu Signed-off-by: bitliu --- .github/workflows/build_and_test.yaml | 4 ++-- .github/workflows/docs.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 1cad2db02d6..ac7d522cb01 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -3,13 +3,13 @@ on: push: branches: - "main" - - "release-v*" + - "release/v*" paths-ignore: - "**/*.png" pull_request: branches: - "main" - - "release-v*" + - "release/v*" paths-ignore: - "**/*.png" jobs: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e070a532ffb..a606203056d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,13 +3,13 @@ on: push: branches: - "main" - - "release-v*" + - "release/v*" paths-ignore: - "**/*.png" pull_request: branches: - "main" - - "release-v*" + - "release/v*" paths-ignore: - "**/*.png" From 34e187e5250247ba151bfd68f7d28c98ed8c5c20 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 20 Oct 2022 09:52:55 -0700 Subject: [PATCH 066/113] Updates Quickstart Guide Links for v0.2.0 Release (#630) Signed-off-by: danehans Signed-off-by: danehans --- docs/user/quickstart.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user/quickstart.md b/docs/user/quickstart.md index 993003e2419..46da189d2ec 100644 --- a/docs/user/quickstart.md +++ b/docs/user/quickstart.md @@ -13,13 +13,13 @@ __Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. Install the Gateway API CRDs and Envoy Gateway: ```shell -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/latest/install.yaml +kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0/install.yaml ``` Install the GatewayClass, Gateway, HTTPRoute and example app: ```shell -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/latest/quickstart.yaml +kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0/quickstart.yaml ``` ## Testing the Configuration @@ -67,13 +67,13 @@ Use the steps in this section to uninstall everything from the quickstart guide. Delete the GatewayClass, Gateway, HTTPRoute and Example App: ```shell -kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/latest/quickstart.yaml --ignore-not-found=true +kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0/quickstart.yaml --ignore-not-found=true ``` Delete the Gateway API CRDs and Envoy Gateway: ```shell -kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/latest/install.yaml --ignore-not-found=true +kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/v0.2.0/install.yaml --ignore-not-found=true ``` ## Next Steps From 5cda0dd621ae00bdc094e8b9034c9dbe760ff8da Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Fri, 21 Oct 2022 01:36:45 +0800 Subject: [PATCH 067/113] Do not allow doc workflow to run when releasing for now (#633) Signed-off-by: bitliu Signed-off-by: bitliu --- .github/workflows/docs.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a606203056d..f8b8fe3696b 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,13 +3,15 @@ on: push: branches: - "main" - - "release/v*" + # Uncomment here until https://github.com/envoyproxy/gateway/issues/632 has been fixed. + # - "release/v*" paths-ignore: - "**/*.png" pull_request: branches: - "main" - - "release/v*" + # Uncomment here until https://github.com/envoyproxy/gateway/issues/632 has been fixed. + # - "release/v*" paths-ignore: - "**/*.png" From 852262af7003b29b177eed2083b17c501226f453 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 20 Oct 2022 12:35:01 -0700 Subject: [PATCH 068/113] Adds v0.2 Release Announcement Doc (#635) * Adds v0.2 Release Announcement Doc Signed-off-by: danehans * Removes RefGrant and Fixes Typo Signed-off-by: danehans Signed-off-by: danehans --- docs/index.rst | 1 + docs/releases/v0.2.md | 50 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 docs/releases/v0.2.md diff --git a/docs/index.rst b/docs/index.rst index 774cfbd653a..42006c0402b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,5 +31,6 @@ standalone or Kubernetes-based application gateway. design/gatewayapi-translator design/watching design/config-api + releases/v0.2 about_docs get_involved diff --git a/docs/releases/v0.2.md b/docs/releases/v0.2.md new file mode 100644 index 00000000000..a0dc0e885de --- /dev/null +++ b/docs/releases/v0.2.md @@ -0,0 +1,50 @@ +--- +title: Announcing Envoy Gateway v0.2 +linktitle: v0.2 +subtitle: Major Update +description: Envoy Gateway v0.2 release announcement. +publishdate: 2022-10-20 +release: v0.2.0 +skip_list: true +aliases: +- /releases/v0.2 +- /releases/v0.2.0 +--- +# Envoy Gateway Release v0.2 + +We are pleased to announce the release of Envoy Gateway v0.2! + +This is the first functional release of Envoy Gateway. We would like to thank the entire Envoy Gateway community for +helping publish the release. + +| [Release Notes][] | [Docs][docs] | [Compatibility Matrix][matrix] | [Download][] | +|-------------------|--------------|--------------------------------|--------------| + +## What's New + +The release adds a ton of features and functionality. Here are some highlights: + +### Kubernetes Support + +Run Envoy Gateway in a Kubernetes cluster. Checkout the [quickstart guide][] to get started with Envoy Gateway in a few +simple steps. + +### Gateway API Support + +Envoy Gateway supports Gateway API resources for running and configuring a managed fleet of Envoy proxies. Envoy Gateway +passes Gateway API core [conformance tests][] and supports GatewayClass, Gateway, HTTPRoute, and TLSRoute resources. See +the [documentation][docs] for additional details on how to use Envoy Gateway for your edge proxy and API gateway needs. + +## Envoy Gateway at EnvoyCon NA + +Envoy Gateway will be at [EnvoyCon NA][] this October in Detroit. Don't miss [our talk][] to learn more about the +release and future direction of the project. + +[Release Notes]: https://github.com/envoyproxy/gateway/blob/main/release-notes/v0.2.0.yaml +[matrix]: https://gateway.envoyproxy.io/intro/compatibility.html +[docs]: https://gateway.envoyproxy.io/index.html +[Download]: https://github.com/envoyproxy/gateway/releases/tag/v0.2.0 +[conformance tests]: https://gateway-api.sigs.k8s.io/concepts/conformance/?h=conformance +[quickstart guide]: https://gateway.envoyproxy.io/user/quickstart.html +[EnvoyCon NA]: https://events.linuxfoundation.org/envoycon-north-america/program/schedule/ +[our talk]: https://sched.co/1AO5S From 362656ce1738bfc29dbb36f489229887e28865e6 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 20 Oct 2022 13:48:30 -0700 Subject: [PATCH 069/113] Reorganizes Sphinx Docs (#636) --- README.md | 4 ++-- docs/about_docs.rst | 6 +++++- docs/design_docs.rst | 12 +++++++++++ .../dev/CODE_OF_CONDUCT.md | 0 CONTRIBUTING.md => docs/dev/CONTRIBUTING.md | 0 DOCS.md => docs/dev/DOCS.md | 0 docs/dev_docs.rst | 12 +++++++++++ docs/index.rst | 21 +++++++------------ docs/releases.rst | 9 ++++++++ docs/roadmap.rst | 9 ++++++++ docs/user_docs.rst | 14 +++++++++++++ 11 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 docs/design_docs.rst rename CODE_OF_CONDUCT.md => docs/dev/CODE_OF_CONDUCT.md (100%) rename CONTRIBUTING.md => docs/dev/CONTRIBUTING.md (100%) rename DOCS.md => docs/dev/DOCS.md (100%) create mode 100644 docs/dev_docs.rst create mode 100644 docs/releases.rst create mode 100644 docs/roadmap.rst create mode 100644 docs/user_docs.rst diff --git a/README.md b/README.md index b48388dc791..6a4e2bd8ff1 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Kubernetes-based application gateway. ## Contributing -* [Code of conduct](CODE_OF_CONDUCT.md) -* [Contributing guide](CONTRIBUTING.md) +* [Code of conduct](./docs/dev/CODE_OF_CONDUCT.md) +* [Contributing guide](./docs/dev/CONTRIBUTING.md) * [Developer guide](docs/dev/README.md) ## Community Meeting diff --git a/docs/about_docs.rst b/docs/about_docs.rst index 7ca351e03ef..64a12d791d7 100644 --- a/docs/about_docs.rst +++ b/docs/about_docs.rst @@ -1,5 +1,9 @@ About the documentation ======================= -The Envoy Gateway documentation is **VERY MUCH A WORK IN PROGRESS**. +Learn how to contribute to Envoy Gateway documentation. +.. toctree:: + :maxdepth: 1 + + dev/DOCS diff --git a/docs/design_docs.rst b/docs/design_docs.rst new file mode 100644 index 00000000000..4e95a518d1e --- /dev/null +++ b/docs/design_docs.rst @@ -0,0 +1,12 @@ +Design Docs +=========== + +Learn about the internal details of Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + design/system-design + design/gatewayapi-translator + design/watching + design/config-api diff --git a/CODE_OF_CONDUCT.md b/docs/dev/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to docs/dev/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/docs/dev/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/dev/CONTRIBUTING.md diff --git a/DOCS.md b/docs/dev/DOCS.md similarity index 100% rename from DOCS.md rename to docs/dev/DOCS.md diff --git a/docs/dev_docs.rst b/docs/dev_docs.rst new file mode 100644 index 00000000000..885b86cd18a --- /dev/null +++ b/docs/dev_docs.rst @@ -0,0 +1,12 @@ +Developer Docs +============== + +Learn how to contribute to Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + dev/CODE_OF_CONDUCT + dev/CONTRIBUTING + dev/README + dev/releasing diff --git a/docs/index.rst b/docs/index.rst index 42006c0402b..bfcb9180980 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,5 @@ Envoy Gateway -======================================= +============= Release |version| (Envoy |envoyVersion|, Gateway API |gatewayAPIVersion|) @@ -16,21 +16,14 @@ standalone or Kubernetes-based application gateway. complete. We would love for you to :doc:`get involved`. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 intro/index intro/compatibility - user/quickstart - user/http-routing - user/http-redirect - user/http-traffic-splitting - user/http-request-headers - user/tls-passthrough - design/system-design - design/roadmap - design/gatewayapi-translator - design/watching - design/config-api - releases/v0.2 + user_docs + design_docs + dev_docs + releases + roadmap about_docs get_involved diff --git a/docs/releases.rst b/docs/releases.rst new file mode 100644 index 00000000000..be29c9e5f9c --- /dev/null +++ b/docs/releases.rst @@ -0,0 +1,9 @@ +Releases +======== + +Learn more about Envoy Gateway releases. + +.. toctree:: + :maxdepth: 2 + + releases/v0.2 diff --git a/docs/roadmap.rst b/docs/roadmap.rst new file mode 100644 index 00000000000..711b6245503 --- /dev/null +++ b/docs/roadmap.rst @@ -0,0 +1,9 @@ +Roadmap +======= + +Learn about the future direction of Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + design/roadmap diff --git a/docs/user_docs.rst b/docs/user_docs.rst new file mode 100644 index 00000000000..c8887e25cd4 --- /dev/null +++ b/docs/user_docs.rst @@ -0,0 +1,14 @@ +User Guides +=========== + +Learn how to deploy, use, and operate Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + user/quickstart + user/http-routing + user/http-redirect + user/http-traffic-splitting + user/http-request-headers + user/tls-passthrough From 3f503723f2a6eedaa20ce8c5200f57087c7be52d Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Fri, 21 Oct 2022 05:44:48 +0800 Subject: [PATCH 070/113] Add support for HTTPQueryParamMatch in Gateway API Translator (#606) Signed-off-by: zhaohuabing --- internal/gatewayapi/helpers.go | 8 ++++++++ .../httproutes-with-multiple-matches.out.yaml | 7 +++---- internal/gatewayapi/translator.go | 8 ++++++++ .../translator/testdata/in/xds-ir/http-route.yaml | 12 +++++++++++- .../testdata/out/xds-ir/http-route.routes.yaml | 10 +++++++++- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index e621c101be8..0cf58152e1e 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -81,6 +81,14 @@ func HeaderMatchTypeDerefOr(matchType *v1beta1.HeaderMatchType, defaultType v1be return defaultType } +func QueryParamMatchTypeDerefOr(matchType *v1beta1.QueryParamMatchType, + defaultType v1beta1.QueryParamMatchType) v1beta1.QueryParamMatchType { + if matchType != nil { + return *matchType + } + return defaultType +} + func NamespaceDerefOr(namespace *v1beta1.Namespace, defaultNamespace string) string { if namespace != nil && *namespace != "" { return string(*namespace) diff --git a/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml b/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml index 60ad2e5736f..c1612da9230 100644 --- a/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml +++ b/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml @@ -187,10 +187,9 @@ xdsIR: - name: envoy-gateway-httproute-2-rule-0-match-0-example.com pathMatch: prefix: "/v1/example" - # Remove comments once https://github.com/envoyproxy/gateway/issues/512 is fixed - # queryParamMatches: - # - name: "debug" - # exact: "yes" + queryParamMatches: + - name: "debug" + exact: "yes" headerMatches: - name: :authority exact: example.com diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 95cc8f23300..2f25b385bd9 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -1075,6 +1075,14 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways }) } } + for _, queryParamMatch := range match.QueryParams { + if QueryParamMatchTypeDerefOr(queryParamMatch.Type, v1beta1.QueryParamMatchExact) == v1beta1.QueryParamMatchExact { + irRoute.QueryParamMatches = append(irRoute.QueryParamMatches, &ir.StringMatch{ + Name: queryParamMatch.Name, + Exact: StringPtr(queryParamMatch.Value), + }) + } + } // Add the redirect filter or direct response that were created earlier to all the irRoutes if redirectResponse != nil { diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route.yaml index 6a8cda1ded7..fd6c9654fbc 100644 --- a/internal/xds/translator/testdata/in/xds-ir/http-route.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/http-route.yaml @@ -5,7 +5,17 @@ http: hostnames: - "*" routes: - - name: "first-route" + - name: "first-route" + pathMatch: + name: "test" + exact: "foo/bar" + headerMatches: + - name: user + stringMatch: + exact: "jason" + queryParamMatches: + - name: "debug" + exact: "yes" destinations: - host: "1.2.3.4" port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml index ed122e552aa..a28539a6d49 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route.routes.yaml @@ -5,6 +5,14 @@ name: first-listener routes: - match: - prefix: / + headers: + - name: user + stringMatch: + exact: jason + path: foo/bar + queryParameters: + - name: debug + stringMatch: + exact: "yes" route: cluster: first-route From 619cbac6795cbe48c15009c2bc20af06ed0a9e23 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Thu, 20 Oct 2022 16:22:58 -0700 Subject: [PATCH 071/113] enable conformance tests for query param match (#637) Relates to https://github.com/envoyproxy/gateway/issues/512 Signed-off-by: Arko Dasgupta --- test/conformance/conformance_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index 8c9730904d8..09f576105b6 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -6,8 +6,6 @@ // The full text of the Apache license is available in the LICENSE file at // the root of the repo. - - package conformance import ( @@ -53,7 +51,7 @@ func TestGatewayAPIConformance(t *testing.T) { Debug: *flags.ShowDebug, CleanupBaseResources: *flags.CleanupBaseResources, ValidUniqueListenerPorts: validUniqueListenerPorts, - SupportedFeatures: []suite.SupportedFeature{suite.SupportReferenceGrant}, + SupportedFeatures: []suite.SupportedFeature{suite.SupportHTTPRouteQueryParamMatching, suite.SupportReferenceGrant}, }) cSuite.Setup(t) egTests := []suite.ConformanceTest{ From 4ad8ff6b692cbfce11710978b516c8efac2dd399 Mon Sep 17 00:00:00 2001 From: Jimmy Song Date: Fri, 21 Oct 2022 13:21:09 +0800 Subject: [PATCH 072/113] docs: update a reference link and doc workflow (#638) --- docs/design/system-design.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/system-design.md b/docs/design/system-design.md index 1f30dc7203e..0e22eb25895 100644 --- a/docs/design/system-design.md +++ b/docs/design/system-design.md @@ -49,7 +49,7 @@ defined as Kubernetes resources that provide the following services: ## Components Envoy Gateway is made up of several components that communicate in-process; how this communication happens is described -in [watching.md][]. +in the [Watching Components Design][wcd]. ### Provider @@ -164,4 +164,4 @@ The draft for this document is [here][draft_design]. [be]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.BackendObjectReference [svc]: https://kubernetes.io/docs/concepts/services-networking/service/ [issue_95]: https://github.com/envoyproxy/gateway/pull/95 -[watching.md]: ./watching.md +[ wcd ]: ./watching.md From 7e9cff94ee6295caf7a79651a686db330b335e22 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Fri, 21 Oct 2022 10:13:07 -0700 Subject: [PATCH 073/113] Updates Release Doc (#634) * Updates Release Doc Signed-off-by: danehans * Adds add'l steps and env vars for releasing Signed-off-by: danehans Signed-off-by: danehans --- docs/dev/releasing.md | 143 ++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 47 deletions(-) diff --git a/docs/dev/releasing.md b/docs/dev/releasing.md index 987aa40b04f..34cc1669452 100644 --- a/docs/dev/releasing.md +++ b/docs/dev/releasing.md @@ -2,84 +2,128 @@ This document guides maintainers through the process of creating an Envoy Gateway release. -## Prerequisites +## Creating a Minor Release + +### Prerequisites - Permissions to push to the Envoy Gateway repository. -## Creating a Minor Release +### Set Environment Variables + +Set environment variables for use in subsequent steps: + +```shell +export MAJOR_VERSION=0 +export MINOR_VERSION=3 +export GITHUB_REMOTE=origin +``` 1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. -2. Create the release notes corresponding to the release number. Reference previous [release notes][] - for additional details. -3. Submit a [Pull Request][] to merge the release notes into the main branch. This should be the last commit to main - before cutting the release. -4. Create a new release branch from `main`. The release branch should be named - `release/v${MAJOR_VERSION}.${MINOR_VERSION}`, e.g. `release/v0.3`. +2. Create a topic branch to create the release notes. Reference previous [release notes][] for additional details. +3. Sign, commit, and push your changes to your fork and submit a [Pull Request][] to merge the release notes into the + `main` branch. +4. Create a topic branch for the release announcement. Reference previous [release announcements][] for additional + details. +5. Sign, commit, and push your changes to your fork and submit a [Pull Request][] to merge the release announcement + into the `main` branch. +6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. This should be the last + commit to main before cutting the release. ```shell - git checkout -b release/v0.3 + make update-quickstart TAG=${MAJOR_VERSION}.${MINOR_VERSION}.0 ``` -5. Push the branch to the Envoy Gateway repo. -6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. +7. Sign, commit, and push your changes to your fork. Send a PR to get your changes merged into main. Do not proceed + until all your PRs have merged and the Build and Test [release GitHub action][] has completed for your final PR. + + __Note:__ The Quickstart update should occur in the release branch after [Issue #632][] is resolved. + +8. Pull the latest changes from the `main` branch that include commits from all the above PRs: ```shell - make update-quickstart TAG=v0.3.0 + git pull ${GITHUB_REMOTE} main ``` -7. Sign, commit, and push your changes to your fork. Send a PR to get your changes merged into the release branch. - Do not proceed until your PR is merged. -8. Tag the head of your release branch with the release tag. For example: +9. Create a new release branch from `main`. The release branch should be named + `release/v${MAJOR_VERSION}.${MINOR_VERSION}`, e.g. `release/v0.3`. ```shell - git tag -a v0.3.0 -m 'Envoy Gateway v0.3.0 Release' + git checkout -b release/v${MAJOR_VERSION}.${MINOR_VERSION} ``` - __Note:__ The tag version differs from the release branch by including the `.0` patch version. +10. Push the branch to the Envoy Gateway repo. + + ```shell + git push ${GITHUB_REMOTE} release/v${MAJOR_VERSION}.${MINOR_VERSION} + ``` -9. Push the tag to the Envoy Gateway repository. +11. Tag the head of your release branch with the release tag. For example: ```shell - git push v0.3.0 + git tag -a v${MAJOR_VERSION}.${MINOR_VERSION}.0 -m 'Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0 Release' ``` -10. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. -11. Confirm that the [release workflow][] completed successfully. -12. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. -13. Confirm that the [release][] was created. -14. Confirm that the steps in the [Quickstart Guide][] work as expected. -15. [Generate][] the GitHub changelog. -16. If you find any bugs in this process, please create an issue. + __Note:__ The tag version differs from the release branch by including the `.0` patch version. + +12. Push the tag to the Envoy Gateway repository. + + ```shell + git push origin v${MAJOR_VERSION}.${MINOR_VERSION}.0 + ``` + +13. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +14. Confirm that the [release workflow][] completed successfully. +15. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +16. Confirm that the [release][] was created. +17. Confirm that the steps in the [Quickstart Guide][] work as expected. +18. [Generate][] the GitHub changelog and include the following text at the beginning of the release page: + + ```console + # Release Announcement + + Check out the [v${MAJOR_VERSION}.${MINOR_VERSION} release announcement] + (https://gateway.envoyproxy.io/releases/v${MAJOR_VERSION}.${MINOR_VERSION}.html) to learn more about the release. + ``` + +19. If you find any bugs in this process, please create an issue. ## Creating a Release Candidate +### Prerequisites + +- Permissions to push to the Envoy Gateway repository. + +### Set Environment Variables + +```shell +export MAJOR_VERSION=0 +export MINOR_VERSION=3 +export RELEASE_CANDIDATE_NUMBER=1 +export GITHUB_REMOTE=origin +``` + 1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. -2. Create the release notes corresponding to the release candidate that summarizes the changes included in the - release candidate. Reference previous [release notes][] for additional details. -3. Submit a [Pull Request][] to merge the changelog into the main branch. This should be the last commit to main - before cutting the release candidate. -4. Tag the head of the main branch with the release candidate number. The tag should be named - `v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER}`. For example: +5. Tag the head of the main branch with the release candidate number. ```shell - git tag -a v0.3.0-rc.1 -m 'Envoy Gateway v0.3.0-rc.1 Release Candidate' + git tag -a v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} -m 'Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} Release Candidate' ``` -5. Push the tag to the Envoy Gateway repository. +6. Push the tag to the Envoy Gateway repository. ```shell - git push v0.3.0-rc.1 + git push v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} ``` -6. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. -7. Confirm that the [release workflow][] completed successfully. -8. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. -9. Confirm that the [release][] was created. -10. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test +7. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +8. Confirm that the [release workflow][] completed successfully. +9. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +10. Confirm that the [release][] was created. +11. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test the quickstart steps using the release candidate by manually updating the links. -11. [Generate][] the GitHub changelog. -12. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. -13. If you find any bugs in this process, please create an issue. +12. [Generate][] the GitHub changelog. +13. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. +14. If you find any bugs in this process, please create an issue. ## Announcing the Release @@ -88,17 +132,20 @@ It's important that the world knows about the release. Use the following steps t 1. Set the release information in the Envoy Gateway Slack channel. For example: ```shell - Envoy Gateway v0.3.0 has been released: https://github.com/envoyproxy/gateway/releases/tag/v0.3.0 + Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION} has been released: https://github.com/envoyproxy/gateway/releases/tag/v${MAJOR_VERSION}.${MINOR_VERSION}.0 ``` 2. Send a message to the Envoy Gateway Slack channel. For example: ```shell - I am pleased to announce the release of Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0. The release would not be - possible without all the support from the Envoy Gateway community... + On behalf of the entire Envoy Gateway community, I am pleased to announce the release of Envoy Gateway + v${MAJOR_VERSION}.${MINOR_VERSION}. A big thank you to all the contributors that made this release possible. + Refer to the official v${MAJOR_VERSION}.${MINOR_VERSION} announcement for release details and the project docs + to start using Envoy Gateway. + ... ``` - Include a sentence or two that highlights key aspects of the release. + Link to the GitHub release and release announcement page that highlights the release. [release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes [Pull Request]: https://github.com/envoyproxy/gateway/pulls @@ -108,3 +155,5 @@ It's important that the world knows about the release. Use the following steps t [image]: https://hub.docker.com/r/envoyproxy/gateway/tags [release]: https://github.com/envoyproxy/gateway/releases [Generate]: https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes +[release announcements]: https://github.com/envoyproxy/gateway/blob/main/docs/releases/v0.2.md +[Issue #632]: https://github.com/envoyproxy/gateway/issues/632 From d572c238cfde8623046de85c06b3139479f1d7aa Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Fri, 21 Oct 2022 13:12:12 -0700 Subject: [PATCH 074/113] Adds Secure Gateway User Doc (#542) --- .gitignore | 5 + docs/user/secure-gateways.md | 260 +++++++++++++++++++++++++++++++++++ docs/user_docs.rst | 1 + 3 files changed, 266 insertions(+) create mode 100644 docs/user/secure-gateways.md diff --git a/.gitignore b/.gitignore index 3dcae053694..02df3075bd3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,8 @@ coverage.xml # `go mod vendor` vendor/ + +# TLS assets that may have been created for secure gateways. +*.crt +*.csr +*.key diff --git a/docs/user/secure-gateways.md b/docs/user/secure-gateways.md new file mode 100644 index 00000000000..e1be52ef111 --- /dev/null +++ b/docs/user/secure-gateways.md @@ -0,0 +1,260 @@ +# Secure Gateways + +This guide will help you get started using secure Gateways. The guide uses a self-signed CA, so it should be used for +testing and demonstration purposes only. + +## Prerequisites + +- A Kubernetes cluster with `kubectl` context configured for the cluster. +- OpenSSL to generate TLS assets. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24. + +## Installation + +Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and the example manifest. +Before proceeding, you should be able to query the example backend using HTTP. + +## TLS Certificates + +Generate the certificates and keys used by the Gateway to terminate client TLS connections. + +Create a root certificate and private key to sign certificates: + +```shell +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=example Inc./CN=example.com' -keyout example.com.key -out example.com.crt +``` + +Create a certificate and a private key for `www.example.com`: + +```shell +openssl req -out www.example.com.csr -newkey rsa:2048 -nodes -keyout www.example.com.key -subj "/CN=www.example.com/O=httpbin organization" +openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in www.example.com.csr -out www.example.com.crt +``` + +Store the cert/key in a Secret: + +```shell +kubectl create secret tls example-cert --key=www.example.com.key --cert=www.example.com.crt +``` + +Update the Gateway from the Quickstart guide to include an HTTPS listener that listens on port `8443` and references the +`example-cert` Secret: + +```shell +kubectl patch gateway eg --type=json --patch '[{ + "op": "add", + "path": "/spec/listeners/-", + "value": { + "name": "https", + "protocol": "HTTPS", + "port": 8443, + "tls": { + "mode": "Terminate", + "certificateRefs": [{ + "kind": "Secret", + "group": "", + "name": "example-cert", + }], + }, + }, +}]' +``` + +Verify the Gateway status: + +```shell +kubectl get gateway/eg -o yaml +``` + +## Testing + +### Clusters without External LoadBalancer Support + +Get the name of the Envoy service created the by the example Gateway: + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8043:8443 & +``` + +Query the example app through Envoy proxy: + +```shell +curl -v -HHost:www.example.com --resolve "www.example.com:8043:127.0.0.1" \ +--cacert example.com.crt https://www.example.com:8043/get +``` + +### Clusters with External LoadBalancer Support + +Get the External IP of the Gateway: + +```shell +export GATEWAY_HOST=$(kubectl get gateway/eg -o jsonpath='{.status.addresses[0].value}') +``` + +Query the example app through the Gateway: + +```shell +curl -v -HHost:www.example.com --resolve "www.example.com:8443:${GATEWAY_HOST}" \ +--cacert example.com.crt https://www.example.com:8443/get +``` + +## Multiple HTTPS Listeners + +Create a TLS cert/key for the additional HTTPS listener: + +```shell +openssl req -out foo.example.com.csr -newkey rsa:2048 -nodes -keyout foo.example.com.key -subj "/CN=foo.example.com/O=httpbin organization" +openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in foo.example.com.csr -out foo.example.com.crt +``` + +Store the cert/key in a Secret: + +```shell +kubectl create secret tls foo-cert --key=foo.example.com.key --cert=foo.example.com.crt +``` + +Create another HTTPS listener on the example Gateway: + +```shell +kubectl patch gateway eg --type=json --patch '[{ + "op": "add", + "path": "/spec/listeners/-", + "value": { + "name": "https-foo", + "protocol": "HTTPS", + "port": 8443, + "hostname": "foo.example.com", + "tls": { + "mode": "Terminate", + "certificateRefs": [{ + "kind": "Secret", + "group": "", + "name": "foo-cert", + }], + }, + }, +}]' +``` + +Update the HTTPRoute to route traffic for hostname `foo.example.com` to the example backend service: + +```shell +kubectl patch httproute backend --type=json --patch '[{ + "op": "add", + "path": "/spec/hostnames/-", + "value": "foo.example.com", +}]' +``` + +Verify the Gateway status: + +```shell +kubectl get gateway/eg -o yaml +``` + +Follow the steps in the [Testing section](#testing) to test connectivity to the backend app through both Gateway +listeners. Replace `www.example.com` with `foo.example.com` to test the new HTTPS listener. + +## Cross Namespace Certificate References + +A Gateway can be configured to reference a certificate in a different namespace. This is allowed by a [ReferenceGrant][] +created in the target namespace. Without the ReferenceGrant, a cross-namespace reference is invalid. + +Before proceeding, ensure you can query the HTTPS backend service from the [Testing section](#testing). + +To demonstrate cross namespace certificate references, create a ReferenceGrant that allows Gateways from the "default" +namespace to reference Secrets in the "envoy-gateway-system" namespace: + +```console +$ cat < Date: Fri, 28 Oct 2022 10:19:11 -0700 Subject: [PATCH 075/113] bring in fixes from go-control-plane (#652) Fixes: https://github.com/envoyproxy/gateway/issues/599 Signed-off-by: Arko Dasgupta --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2f2105e58e5..38d3b23721e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/envoyproxy/gateway go 1.18 require ( - github.com/envoyproxy/go-control-plane v0.10.3-0.20220719090109-b024c36d9935 + github.com/envoyproxy/go-control-plane v0.10.3-0.20221028143534-ed9652aebfd9 github.com/go-logr/zapr v1.2.0 github.com/google/go-cmp v0.5.8 github.com/spf13/cobra v1.4.0 @@ -71,7 +71,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 6d05c0a69a7..e6c97b8b3e5 100644 --- a/go.sum +++ b/go.sum @@ -144,8 +144,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/go-control-plane v0.10.3-0.20220719090109-b024c36d9935 h1:1P6HktLf+VNpEwASft2E0KU7ddeuu73UMnFpawKuD58= -github.com/envoyproxy/go-control-plane v0.10.3-0.20220719090109-b024c36d9935/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.10.3-0.20221028143534-ed9652aebfd9 h1:FT3IU5c8nN3lMfhMVsYeV/CR7TiqtpASnF/7dE6vwnc= +github.com/envoyproxy/go-control-plane v0.10.3-0.20221028143534-ed9652aebfd9/go.mod h1:ufpOdMVWU+v42FYQiIBUhSWglFcK3S1Ml8bbzLwkdcE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7 h1:qcZcULcd/abmQg6dwigimCNEyi4gg31M/xaciQlDml8= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= @@ -419,8 +419,9 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= From 6f540b51db45fefd95ca13f6f9ec9f835c3c3f7d Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 28 Oct 2022 11:24:58 -0600 Subject: [PATCH 076/113] internal/gatewayapi: decouple test cases from default proxy image (#653) Adds an optional proxy image override to the translator so that test cases can be decoupled from the default image, which will change over time. Closes #346. Signed-off-by: Steve Kriss --- ...ows-same-namespace-with-allowed-httproute.out.yaml | 2 +- ...-same-namespace-with-disallowed-httproute.out.yaml | 2 +- ...-with-invalid-allowed-namespaces-selector.out.yaml | 2 +- ...istener-with-invalid-allowed-routes-group.out.yaml | 2 +- ...listener-with-invalid-allowed-routes-kind.out.yaml | 2 +- ...tener-with-invalid-allowed-tls-route-kind.out.yaml | 2 +- ...th-invalid-tls-configuration-invalid-mode.out.yaml | 2 +- ...lid-tls-configuration-no-certificate-refs.out.yaml | 2 +- ...d-tls-configuration-secret-does-not-exist.out.yaml | 2 +- ...s-configuration-secret-in-other-namespace.out.yaml | 2 +- ...lid-tls-configuration-secret-is-not-valid.out.yaml | 2 +- ...-with-missing-allowed-namespaces-selector.out.yaml | 2 +- ...et-in-other-namespace-allowed-by-refgrant.out.yaml | 2 +- ...stener-with-tls-terminate-and-passthrough.out.yaml | 2 +- ...y-with-listener-with-unsupported-protocol.out.yaml | 2 +- ...ith-listener-with-valid-tls-configuration.out.yaml | 2 +- ...-http-and-tlsroute-same-hostname-and-port.out.yaml | 2 +- ...two-listeners-with-same-port-and-hostname.out.yaml | 2 +- ...-with-same-port-and-incompatible-protocol.out.yaml | 2 +- ...-to-gateway-with-more-different-listeners.out.yaml | 2 +- ...-attaching-to-gateway-with-more-listeners.out.yaml | 2 +- ...y-with-two-listeners-with-different-ports.out.yaml | 2 +- ...e-attaching-to-gateway-with-two-listeners.out.yaml | 2 +- .../testdata/httproute-attaching-to-gateway.out.yaml | 2 +- ...to-listener-on-gateway-with-two-listeners.out.yaml | 2 +- .../testdata/httproute-attaching-to-listener.out.yaml | 2 +- ...ule-with-multiple-backends-and-no-weights.out.yaml | 2 +- ...e-rule-with-multiple-backends-and-weights.out.yaml | 2 +- ...ef-in-other-namespace-allowed-by-refgrant.out.yaml | 2 +- ...der-filter-duplicate-add-multiple-filters.out.yaml | 2 +- ...tproute-with-header-filter-duplicate-adds.out.yaml | 2 +- ...-filter-duplicate-remove-multiple-filters.out.yaml | 2 +- ...oute-with-header-filter-duplicate-removes.out.yaml | 2 +- ...te-with-header-filter-empty-header-values.out.yaml | 2 +- ...ttproute-with-header-filter-empty-headers.out.yaml | 2 +- ...proute-with-header-filter-invalid-headers.out.yaml | 2 +- .../httproute-with-header-filter-no-headers.out.yaml | 2 +- ...route-with-header-filter-no-valid-headers.out.yaml | 2 +- .../httproute-with-header-filter-remove.out.yaml | 2 +- ...tproute-with-invalid-backend-ref-bad-port.out.yaml | 2 +- ...te-with-invalid-backend-ref-invalid-group.out.yaml | 2 +- ...ute-with-invalid-backend-ref-invalid-kind.out.yaml | 2 +- ...ttproute-with-invalid-backend-ref-no-port.out.yaml | 2 +- ...route-with-invalid-backend-ref-no-service.out.yaml | 2 +- ...ith-invalid-backendref-in-other-namespace.out.yaml | 2 +- ...taching-to-gateway-with-wildcard-hostname.out.yaml | 2 +- ...h-redirect-filter-full-path-replace-https.out.yaml | 2 +- .../httproute-with-redirect-filter-hostname.out.yaml | 2 +- ...-with-redirect-filter-invalid-filter-type.out.yaml | 2 +- ...route-with-redirect-filter-invalid-scheme.out.yaml | 2 +- ...route-with-redirect-filter-invalid-status.out.yaml | 2 +- ...rect-filter-prefix-replace-with-port-http.out.yaml | 2 +- ...te-with-single-rule-with-exact-path-match.out.yaml | 2 +- ...with-path-prefix-and-exact-header-matches.out.yaml | 2 +- ...with-some-invalid-backend-refs-no-service.out.yaml | 2 +- ...taching-to-gateway-with-wildcard-hostname.out.yaml | 2 +- ...taching-to-gateway-with-wildcard-hostname.out.yaml | 2 +- ...ith-urlrewrite-filter-invalid-filter-type.out.yaml | 2 +- .../httproutes-with-multiple-matches.out.yaml | 2 +- .../testdata/tlsroute-attaching-to-gateway.out.yaml | 2 +- ...-attaching-to-gateway-with-incorrect-mode.out.yaml | 2 +- ...ute-not-attaching-to-gateway-with-no-mode.out.yaml | 2 +- ...ef-in-other-namespace-allowed-by-refgrant.out.yaml | 2 +- ...h-listener-both-passthrough-and-cert-data.out.yaml | 2 +- internal/gatewayapi/translator.go | 11 +++++++++++ internal/gatewayapi/translator_test.go | 1 + 66 files changed, 76 insertions(+), 64 deletions(-) diff --git a/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-allowed-httproute.out.yaml b/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-allowed-httproute.out.yaml index 4ec6b5e9f85..7dd8df241ee 100644 --- a/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-allowed-httproute.out.yaml +++ b/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-allowed-httproute.out.yaml @@ -77,7 +77,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-disallowed-httproute.out.yaml b/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-disallowed-httproute.out.yaml index ef0063b5b3a..564ce5e3385 100644 --- a/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-disallowed-httproute.out.yaml +++ b/internal/gatewayapi/testdata/gateway-allows-same-namespace-with-disallowed-httproute.out.yaml @@ -69,7 +69,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-namespaces-selector.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-namespaces-selector.out.yaml index c9df1469eac..786aa790a33 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-namespaces-selector.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-namespaces-selector.out.yaml @@ -69,6 +69,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-group.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-group.out.yaml index 520a94d69cb..b7dbf3cca34 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-group.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-group.out.yaml @@ -67,6 +67,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-kind.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-kind.out.yaml index 140cb3dc8b9..24da45d907c 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-kind.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-routes-kind.out.yaml @@ -67,6 +67,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml index 59c61cc7162..e4c95db8bb5 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-allowed-tls-route-kind.out.yaml @@ -67,6 +67,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml index 13c19dc555f..02d5aead369 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-invalid-mode.out.yaml @@ -68,6 +68,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-no-certificate-refs.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-no-certificate-refs.out.yaml index 7f9685d1c08..29923e8a403 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-no-certificate-refs.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-no-certificate-refs.out.yaml @@ -65,6 +65,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-does-not-exist.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-does-not-exist.out.yaml index 3219789b684..014573079c1 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-does-not-exist.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-does-not-exist.out.yaml @@ -71,6 +71,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml index 757e4c695e9..af0ec8df8d2 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-in-other-namespace.out.yaml @@ -72,6 +72,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-is-not-valid.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-is-not-valid.out.yaml index 6c1dc8b1550..23868e2b77d 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-is-not-valid.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-invalid-tls-configuration-secret-is-not-valid.out.yaml @@ -71,6 +71,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-missing-allowed-namespaces-selector.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-missing-allowed-namespaces-selector.out.yaml index 54e12a52725..edbca3ecb79 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-missing-allowed-namespaces-selector.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-missing-allowed-namespaces-selector.out.yaml @@ -63,6 +63,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-secret-in-other-namespace-allowed-by-refgrant.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-secret-in-other-namespace-allowed-by-refgrant.out.yaml index 910d29210c6..90b8679b1c9 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-secret-in-other-namespace-allowed-by-refgrant.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-secret-in-other-namespace-allowed-by-refgrant.out.yaml @@ -85,7 +85,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml index f47a7e3a1f2..26002751f1e 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml @@ -140,7 +140,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-unsupported-protocol.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-unsupported-protocol.out.yaml index a5a2dff4287..2ffb00f9481 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-unsupported-protocol.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-unsupported-protocol.out.yaml @@ -64,6 +64,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml index ecd190d5e18..f51684e2a0e 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-valid-tls-configuration.out.yaml @@ -84,7 +84,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml index f0c6ae9bc5f..975d8f585bc 100644 --- a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-http-and-tlsroute-same-hostname-and-port.out.yaml @@ -115,6 +115,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-hostname.out.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-hostname.out.yaml index b2d6d8a773c..b8319813128 100644 --- a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-hostname.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-hostname.out.yaml @@ -87,6 +87,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-incompatible-protocol.out.yaml b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-incompatible-protocol.out.yaml index 66d91bfb60f..3ab2adbf4c3 100644 --- a/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-incompatible-protocol.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-two-listeners-with-same-port-and-incompatible-protocol.out.yaml @@ -87,6 +87,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml index c93b03ed7df..acd287a9516 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-different-listeners.out.yaml @@ -288,7 +288,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml index b0c96ad796f..6fea241c693 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-more-listeners.out.yaml @@ -288,7 +288,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners-with-different-ports.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners-with-different-ports.out.yaml index 8e1de469390..b641ae2d49a 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners-with-different-ports.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners-with-different-ports.out.yaml @@ -113,7 +113,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners.out.yaml index 32bcf4d8409..ca7d3c917a0 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway-with-two-listeners.out.yaml @@ -108,7 +108,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-gateway.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-gateway.out.yaml index 03294c5496f..2e521d61675 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-gateway.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-gateway.out.yaml @@ -77,7 +77,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-listener-on-gateway-with-two-listeners.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-listener-on-gateway-with-two-listeners.out.yaml index 29157f1eeb4..9d0ebd76c9b 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-listener-on-gateway-with-two-listeners.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-listener-on-gateway-with-two-listeners.out.yaml @@ -102,7 +102,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-attaching-to-listener.out.yaml b/internal/gatewayapi/testdata/httproute-attaching-to-listener.out.yaml index 6e384d6ba51..071675942cc 100644 --- a/internal/gatewayapi/testdata/httproute-attaching-to-listener.out.yaml +++ b/internal/gatewayapi/testdata/httproute-attaching-to-listener.out.yaml @@ -79,7 +79,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-no-weights.out.yaml b/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-no-weights.out.yaml index 907102e5e29..be8b2b1bb16 100644 --- a/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-no-weights.out.yaml +++ b/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-no-weights.out.yaml @@ -87,7 +87,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-weights.out.yaml b/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-weights.out.yaml index 98233570d75..dd2c8c35d14 100644 --- a/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-weights.out.yaml +++ b/internal/gatewayapi/testdata/httproute-rule-with-multiple-backends-and-weights.out.yaml @@ -90,7 +90,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml b/internal/gatewayapi/testdata/httproute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml index 6f2809ff6e8..379aa0c1aa2 100644 --- a/internal/gatewayapi/testdata/httproute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml @@ -79,7 +79,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml index 10a378a589b..f7162af9ab4 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-add-multiple-filters.out.yaml @@ -110,7 +110,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml index c23b5425678..54ab4403f29 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-adds.out.yaml @@ -126,7 +126,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml index 01c33fa8d5e..14906d98cf2 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-remove-multiple-filters.out.yaml @@ -100,7 +100,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml index bde4b77b6f7..6e551f461b5 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-duplicate-removes.out.yaml @@ -93,7 +93,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-header-values.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-header-values.out.yaml index aa8c5e0ceff..372e90ebf7a 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-header-values.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-header-values.out.yaml @@ -101,7 +101,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml index cdf6c1f2f49..b35d829de07 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-empty-headers.out.yaml @@ -100,7 +100,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml index 2aec958edfb..0551d6c7076 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-invalid-headers.out.yaml @@ -100,7 +100,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-no-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-no-headers.out.yaml index d7e95be2b3e..daace524950 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-no-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-no-headers.out.yaml @@ -91,7 +91,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml index 399ca601319..31ef9cc2688 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-no-valid-headers.out.yaml @@ -91,7 +91,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-header-filter-remove.out.yaml b/internal/gatewayapi/testdata/httproute-with-header-filter-remove.out.yaml index 40d0556f5ef..ca5123dd3cd 100644 --- a/internal/gatewayapi/testdata/httproute-with-header-filter-remove.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-header-filter-remove.out.yaml @@ -96,7 +96,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml index 3218c2096ed..3c277285a9e 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-bad-port.out.yaml @@ -82,7 +82,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-group.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-group.out.yaml index 92d17de3237..5b09850fa10 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-group.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-group.out.yaml @@ -84,7 +84,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml index 7b0ad031a99..548e238ba82 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml @@ -83,7 +83,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml index 5e41249f9d5..a9cf51c095b 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-port.out.yaml @@ -81,7 +81,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml index 27f0f389c47..8aabd05f3a1 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-no-service.out.yaml @@ -82,7 +82,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backendref-in-other-namespace.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backendref-in-other-namespace.out.yaml index 1ffa12050dd..ed769a16a6f 100644 --- a/internal/gatewayapi/testdata/httproute-with-invalid-backendref-in-other-namespace.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-invalid-backendref-in-other-namespace.out.yaml @@ -83,7 +83,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-non-matching-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml b/internal/gatewayapi/testdata/httproute-with-non-matching-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml index 3bb183f1475..b9baf17ed99 100644 --- a/internal/gatewayapi/testdata/httproute-with-non-matching-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-non-matching-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml @@ -72,7 +72,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-full-path-replace-https.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-full-path-replace-https.out.yaml index 3d42bf63d9e..a8f3819cce4 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-full-path-replace-https.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-full-path-replace-https.out.yaml @@ -94,7 +94,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-hostname.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-hostname.out.yaml index 79ffd78cdd0..e516810982c 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-hostname.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-hostname.out.yaml @@ -91,7 +91,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml index 0217bbf5716..b9cea0da772 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-filter-type.out.yaml @@ -92,7 +92,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml index c211bfb8e69..9ca16051d6b 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-scheme.out.yaml @@ -92,7 +92,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml index 007254e44ab..e9a96d4294c 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-invalid-status.out.yaml @@ -92,7 +92,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-redirect-filter-prefix-replace-with-port-http.out.yaml b/internal/gatewayapi/testdata/httproute-with-redirect-filter-prefix-replace-with-port-http.out.yaml index 59361b9ede0..4b8f4d89afb 100644 --- a/internal/gatewayapi/testdata/httproute-with-redirect-filter-prefix-replace-with-port-http.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-redirect-filter-prefix-replace-with-port-http.out.yaml @@ -96,7 +96,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-single-rule-with-exact-path-match.out.yaml b/internal/gatewayapi/testdata/httproute-with-single-rule-with-exact-path-match.out.yaml index 0e074c278b5..cb0cde346dc 100644 --- a/internal/gatewayapi/testdata/httproute-with-single-rule-with-exact-path-match.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-single-rule-with-exact-path-match.out.yaml @@ -78,7 +78,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-single-rule-with-path-prefix-and-exact-header-matches.out.yaml b/internal/gatewayapi/testdata/httproute-with-single-rule-with-path-prefix-and-exact-header-matches.out.yaml index cc0b4812552..851b5c7802e 100644 --- a/internal/gatewayapi/testdata/httproute-with-single-rule-with-path-prefix-and-exact-header-matches.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-single-rule-with-path-prefix-and-exact-header-matches.out.yaml @@ -87,7 +87,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-some-invalid-backend-refs-no-service.out.yaml b/internal/gatewayapi/testdata/httproute-with-some-invalid-backend-refs-no-service.out.yaml index 0a657f4ffd7..c1e7d30a17b 100644 --- a/internal/gatewayapi/testdata/httproute-with-some-invalid-backend-refs-no-service.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-some-invalid-backend-refs-no-service.out.yaml @@ -89,7 +89,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml b/internal/gatewayapi/testdata/httproute-with-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml index feb6375de05..2cb39d4f047 100644 --- a/internal/gatewayapi/testdata/httproute-with-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-specific-hostname-attaching-to-gateway-with-wildcard-hostname.out.yaml @@ -83,7 +83,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-two-specific-hostnames-attaching-to-gateway-with-wildcard-hostname.out.yaml b/internal/gatewayapi/testdata/httproute-with-two-specific-hostnames-attaching-to-gateway-with-wildcard-hostname.out.yaml index f632488ce31..2a203e13f4b 100644 --- a/internal/gatewayapi/testdata/httproute-with-two-specific-hostnames-attaching-to-gateway-with-wildcard-hostname.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-two-specific-hostnames-attaching-to-gateway-with-wildcard-hostname.out.yaml @@ -94,7 +94,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml b/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml index 8ff359b4867..6e6ac4f447d 100644 --- a/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-urlrewrite-filter-invalid-filter-type.out.yaml @@ -90,7 +90,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml b/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml index c1612da9230..1c300b1ad5a 100644 --- a/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml +++ b/internal/gatewayapi/testdata/httproutes-with-multiple-matches.out.yaml @@ -244,7 +244,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway gateway.envoyproxy.io/owning-gateway-name: gateway-1 name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml index 9a82becdbce..63eeb40cacd 100644 --- a/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml @@ -74,7 +74,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml index 90c62500616..59d04cad1af 100644 --- a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-incorrect-mode.out.yaml @@ -62,6 +62,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml index dbc22d0a5d5..a055531eac7 100644 --- a/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-not-attaching-to-gateway-with-no-mode.out.yaml @@ -60,6 +60,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml index 9bc5dc2c00d..65926af88cb 100644 --- a/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml @@ -75,7 +75,7 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" ports: diff --git a/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml index b8b13220444..34c53240043 100644 --- a/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-with-listener-both-passthrough-and-cert-data.out.yaml @@ -65,6 +65,6 @@ infraIR: gateway.envoyproxy.io/owning-gateway-name: gateway-1 gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway name: envoy-gateway-gateway-1 - image: envoyproxy/envoy:v1.23-latest + image: envoyproxy/envoy:translator-tests listeners: - address: "" diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 2f25b385bd9..a01bf1ba794 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -91,7 +91,14 @@ func (r *Resources) GetSecret(namespace, name string) *v1.Secret { // Translator translates Gateway API resources to IRs and computes status // for Gateway API resources. type Translator struct { + // GatewayClassName is the name of the GatewayClass + // to process Gateways for. GatewayClassName v1beta1.ObjectName + + // ProxyImage is the optional proxy image to use in + // the Infra IR. If unspecified, the default proxy + // image will be used. + ProxyImage string } type TranslateResult struct { @@ -252,6 +259,10 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap gwInfraIR := ir.NewInfra() gwInfraIR.Proxy.Name = irKey gwInfraIR.Proxy.GetProxyMetadata().Labels = GatewayOwnerLabels(gateway.Namespace, gateway.Name) + if len(t.ProxyImage) > 0 { + gwInfraIR.Proxy.Image = t.ProxyImage + } + // save the IR references in the map before the translation starts xdsIR[irKey] = gwXdsIR infraIR[irKey] = gwInfraIR diff --git a/internal/gatewayapi/translator_test.go b/internal/gatewayapi/translator_test.go index a343623537f..5615ab0b83a 100644 --- a/internal/gatewayapi/translator_test.go +++ b/internal/gatewayapi/translator_test.go @@ -48,6 +48,7 @@ func TestTranslate(t *testing.T) { translator := &Translator{ GatewayClassName: "envoy-gateway-class", + ProxyImage: "envoyproxy/envoy:translator-tests", } // Add common test fixtures From 09155a96662da1407527ab9ec72384eccaa78415 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Fri, 28 Oct 2022 11:51:09 -0700 Subject: [PATCH 077/113] Updates Design Docs for v0.2 (#645) Signed-off-by: danehans --- docs/design/config-api.md | 12 +- docs/design/gatewayapi-translator.md | 173 ++++++++++++++------------- docs/design/system-design.md | 58 +++++---- docs/design/watching.md | 2 +- 4 files changed, 128 insertions(+), 117 deletions(-) diff --git a/docs/design/config-api.md b/docs/design/config-api.md index 5a8478d8df8..3696860dd54 100644 --- a/docs/design/config-api.md +++ b/docs/design/config-api.md @@ -46,7 +46,7 @@ The `v1alpha1` version and `config.gateway.envoyproxy.io` API group get generate package v1alpha1 ``` -The initial `EnvoyGateway` API being proposed: +The initial `EnvoyGateway` API: ```go // gateway/api/config/v1alpha1/envoygateway.go @@ -231,17 +231,17 @@ $ ./envoy-gateway The data plane is configured dynamically through Kubernetes resources, primarily [Gateway API][gw_api] objects. Optionally, the data plane infrastructure can be configured by referencing a [custom resource (CR)][cr] through -`spec.parametersRef` of the managed GatewayClass. The `envoyproxies` API defines the data plane infrastructure +`spec.parametersRef` of the managed GatewayClass. The `EnvoyProxy` API defines the data plane infrastructure configuration and is represented as the CR referenced by the managed GatewayClass. Key points of this API are: -* If unreferenced by `spec.parametersRef`, default parameters will be used to configure the data plane infrastructure, - e.g. expose Envoy network endpoints using a LoadBalancer service. +* If unreferenced by `gatewayclass.spec.parametersRef`, default parameters will be used to configure the data plane + infrastructure, e.g. expose Envoy network endpoints using a LoadBalancer service. * Envoy Gateway will follow Gateway API [recommendations][gc] regarding updates to the EnvoyProxy CR: > It is recommended that this resource be used as a template for Gateways. This means that a Gateway is based on the > state of the GatewayClass at the time it was created and changes to the GatewayClass or associated parameters are > not propagated down to existing Gateways. -The initial `envoyproxies` API being proposed: +The initial `EnvoyProxy` API: ```go // gateway/api/config/v1alpha1/envoyproxy.go @@ -302,7 +302,7 @@ spec: ``` Since the GatewayClass does not define `spec.parametersRef`, the data plane is provisioned using default configuration -parameters. All Envoy proxies will be configured with a http listener and a Kubernetes LoadBalancer service listening +parameters. The Envoy proxies will be configured with a http listener and a Kubernetes LoadBalancer service listening on port 80. The following example will configure the data plane to use a ClusterIP service instead of the default LoadBalancer diff --git a/docs/design/gatewayapi-translator.md b/docs/design/gatewayapi-translator.md index 5f2d4abcffa..cd09a8b6c73 100644 --- a/docs/design/gatewayapi-translator.md +++ b/docs/design/gatewayapi-translator.md @@ -1,35 +1,38 @@ # Gateway API Translator Design +The Gateway API translates external resources, e.g. GatewayClass, from the configured Provider to the Intermediate +Representation (IR). + ## Assumptions -- initially target core conformance features only, to be followed by extended conformance features +Initially target core conformance features only, to be followed by extended conformance features. ## Inputs and Outputs The main inputs to the Gateway API translator are: -- the GatewayClass to process -- Gateways, HTTPRoutes, Services, Secrets +- GatewayClass, Gateway, HTTPRoute, TLSRoute, Service, ReferenceGrant, Namespace, and Secret resources. + +__Note:__ ReferenceGrant is not fully implemented as of v0.2. The outputs of the Gateway API translator are: -- IR -- status updates for GatewayClass, Gateways, HTTPRoutes +- Xds and Infra Internal Representations (IRs). +- Status updates for GatewayClass, Gateways, HTTPRoutes ## Listener Compatibility -Since Envoy Gateway handles all Gateways for a given GatewayClass, we need to determine the compatibility of _all_ Listeners across _all_ of those Gateways. - -The rules are: -- for a given port number, every Listener using that port number must have a compatible protocol (either all HTTP, or all HTTPS/TLS). -- for a given port number, every Listener using that port number must have a distinct hostname (at most one Listener per port can have no hostname). +Envoy Gateway follows Gateway API listener compatibility spec: +> Each listener in a Gateway must have a unique combination of Hostname, Port, and Protocol. An implementation MAY group +> Listeners by Port and then collapse each group of Listeners into a single Listener if the implementation determines +> that the Listeners in the group are “compatible”. -Listeners sharing a port that are not mutually compatible will be marked as "Conflicted: true" with an appropriate reason. +__Note:__ Envoy Gateway does not collapse listeners across multiple Gateways. ### Listener Compatibility Examples -#### Example 1: Gateways with compatible Listeners (same port & protocol, different hostnames) +#### Example 1: Gateway with compatible Listeners (same port & protocol, different hostnames) ```yaml kind: Gateway @@ -47,15 +50,6 @@ spec: namespaces: from: All hostname: *.envoygateway.io ---- -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1beta1 -metadata: - name: gateway-2 - namespace: envoy-gateway -spec: - gatewayClassName: envoy-gateway - listeners: - name: http protocol: HTTP port: 80 @@ -65,7 +59,7 @@ spec: hostname: whales.envoygateway.io ``` -#### Example 2: Gateways with compatible Listeners (same port & protocol, one hostname specified, one not) +#### Example 2: Gateway with compatible Listeners (same port & protocol, one hostname specified, one not) ```yaml kind: Gateway @@ -83,15 +77,6 @@ spec: namespaces: from: All hostname: *.envoygateway.io ---- -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1beta1 -metadata: - name: gateway-2 - namespace: envoy-gateway -spec: - gatewayClassName: envoy-gateway - listeners: - name: http protocol: HTTP port: 80 @@ -100,7 +85,7 @@ spec: from: All ``` -#### Example 3: Gateways with incompatible Listeners (same port, protocol and hostname) +#### Example 3: Gateway with incompatible Listeners (same port, protocol and hostname) ```yaml kind: Gateway @@ -118,15 +103,6 @@ spec: namespaces: from: All hostname: whales.envoygateway.io ---- -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1beta1 -metadata: - name: gateway-2 - namespace: envoy-gateway -spec: - gatewayClassName: envoy-gateway - listeners: - name: http protocol: HTTP port: 80 @@ -136,7 +112,7 @@ spec: hostname: whales.envoygateway.io ``` -#### Example 4: Gateways with incompatible Listeners (neither specify a hostname) +#### Example 4: Gateway with incompatible Listeners (neither specify a hostname) ```yaml kind: Gateway @@ -153,15 +129,6 @@ spec: allowedRoutes: namespaces: from: All ---- -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1beta1 -metadata: - name: gateway-2 - namespace: envoy-gateway -spec: - gatewayClassName: envoy-gateway - listeners: - name: http protocol: HTTP port: 80 @@ -172,28 +139,32 @@ spec: ## Computing Status -Gateway API specifies a rich set of status fields & conditions for each resource. -To be conformant, Envoy Gateway needs to compute the appropriate status fields and conditions as it's processing resources. +Gateway API specifies a rich set of status fields & conditions for each resource. To achieve conformance, Envoy Gateway +must compute the appropriate status fields and conditions for managed resources. -Status needs to be computed and set for: +Status is computed and set for: -- the GatewayClass (gatewayclass.status.conditions) -- each Listener for each Gateway (gateway.status.listeners) -- each Gateway, based on its Listeners' statuses (gateway.status.conditions) -- each ParentRef for each Route (route.status.parents) +- The managed GatewayClass (`gatewayclass.status.conditions`). +- Each managed Gateway, based on its Listeners' status (`gateway.status.conditions`). For the Kubernetes provider, the + Envoy Deployment and Service status are also included to calculate Gateway status. +- Listeners for each Gateway (`gateway.status.listeners`). +- The ParentRef for each Route (`route.status.parents`). -The Gateway API translator will take the approach of populating status on the resources themselves as they're being processed, and then passing those statuses off to another component to persist the updates to the Kubernetes API or other backend. +The Gateway API translator is responsible for calculating status conditions while translating Gateway API resources to +the IR and publishing status over the [message bus][]. The Status Manager subscribes to these status messages and +updates the resource status using the configured provider. For example, the Status Manager uses a Kubernetes client to +update resource status on the Kubernetes API server. ## Outline -The following roughly outlines the translation process. -Each step may produce (1) IR; and (2) status updates on Gateway API resources. +The following roughly outlines the translation process. Each step may produce (1) IR; and (2) status updates on Gateway +API resources. 1. Process Gateway Listeners - - validate unique hostnames/ports/protcols - - validate/compute supported kinds - - validate allowed namespaces (validate selector if specified) - - validate TLS details if specified, resolve secret ref + - Validate unique hostnames, ports, and protocols. + - Validate and compute supported kinds. + - Validate allowed namespaces (validate selector if specified). + - Validate TLS fields if specified, including resolving referenced Secrets. 2. Process HTTPRoutes - foreach route rule: @@ -216,31 +187,65 @@ Each step may produce (1) IR; and (2) status updates on Gateway API resources. ## Context Structs -To help store, access and manipulate information as it's processed during the translation process, a set of context structs will be used. -These structs will wrap a given Gateway API type, and add additional fields and methods to support processing. -For example, below is a partial sketch of the ListenerContext struct: +To help store, access and manipulate information as it's processed during the translation process, a set of context +structs are used. These structs wrap a given Gateway API type, and add additional fields and methods to support +processing. + +`GatewayContext` wraps a Gateway and provides helper methods for setting conditions, accessing Listeners, etc. ```go -type listenerContext struct { - // The Listener. - listener *gatewayapi_v1beta1.Listener - - // The Gateway this Listener belongs to. - gateway *gatewayapi_v1beta1.Gateway - - // The TLS Secret for this Listener, if applicable. - tlsSecret *corev1.Secret +type GatewayContext struct { + // The managed Gateway + *v1beta1.Gateway + + // A list of Gateway ListenerContexts. + listeners []*ListenerContext } +``` + +`ListenerContext` wraps a Listener and provides helper methods for setting conditions and other status information on +the associated Gateway. -// Sets a Listener condition on the Listener's Gateway's .status.listeners. -func (lctx *ListenerContext) SetCondition(type string, status bool, reason string, message string) { - ... +```go +type ListenerContext struct { + // The Gateway listener. + *v1beta1.Listener + + // The Gateway this Listener belongs to. + gateway *v1beta1.Gateway + + // An index used for managing this listener in the list of Gateway listeners. + listenerStatusIdx int + + // Only Routes in namespaces selected by the selector may be attached + // to the Gateway this listener belongs to. + namespaceSelector labels.Selector + + // The TLS Secret for this Listener, if applicable. + tlsSecret *v1.Secret } +``` + +`RouteContext` represents a generic Route object (HTTPRoute, TLSRoute, etc.) that can reference Gateway objects. + +```go +type RouteContext interface { + client.Object + + // GetRouteType returns the Kind of the Route object, HTTPRoute, + // TLSRoute, TCPRoute, UDPRoute etc. + GetRouteType() string + + // GetHostnames returns the hosts targeted by the Route object. + GetHostnames() []string + + // GetParentReferences returns the ParentReference of the Route object. + GetParentReferences() []v1beta1.ParentReference -// Returns whether or not the Listener allows a given Route kind. -func (lctx *ListenerContext) AllowsKind(kind gatewayapi_v1beta1.Kind) bool { - ... + // GetRouteParentContext returns RouteParentContext by using the Route + // objects' ParentReference. + GetRouteParentContext(forParentRef v1beta1.ParentReference) *RouteParentContext } ``` -The exact specs of these structs will be worked out at implementation time. +[message bus]: watching.md diff --git a/docs/design/system-design.md b/docs/design/system-design.md index 0e22eb25895..96747c0b27a 100644 --- a/docs/design/system-design.md +++ b/docs/design/system-design.md @@ -28,7 +28,7 @@ Kubernetes resources, primarily [Gateway API][gw_api] objects. Static configuration is used to configure Envoy Gateway at startup, i.e. change the GatewayClass controllerName, configure a Provider, etc. Currently, Envoy Gateway only supports configuration through a configuration file. If the -configuration file is not provided, Envoy Gateway will start up with default configuration parameters. +configuration file is not provided, Envoy Gateway starts-up with default configuration parameters. ### Dynamic Configuration @@ -37,8 +37,8 @@ reconciliation loops to drive the actual state toward the desired state. The des defined as Kubernetes resources that provide the following services: * Infrastructure Management- Manage the data plane infrastructure, i.e. deploy, upgrade, etc. This configuration is - expressed through [GatewayClass][gc] and [Gateway][gw] resources. A TBD [Custom Resource][cr] can be referenced by - `gatewayclass.spec.parametersRef` to modify data plane infrastructure default parameters, + expressed through [GatewayClass][gc] and [Gateway][gw] resources. The `EnvoyProxy` [Custom Resource][cr] can be + referenced by `gatewayclass.spec.parametersRef` to modify data plane infrastructure default parameters, e.g. expose Envoy network endpoints using a NodePort service instead of a LoadBalancer service. * Traffic Routing- Define how to handle application-level requests to backend services. For example, route all HTTP requests for "www.example.com" to a backend service running a web server. This configuration is expressed through @@ -54,9 +54,9 @@ in the [Watching Components Design][wcd]. ### Provider A Provider is an infrastructure component that Envoy Gateway calls to establish its runtime configuration, resolve -services, persist data, etc. Kubernetes and File are the only supported providers. However, other providers can be added -in the future as Envoy Gateway use cases are better understood. A provider is configured at start up through Envoy -Gateway's [static configuration](#static-configuration). +services, persist data, etc. As of v0.2, Kubernetes is the only implemented provider. A file provider is on the roadmap +via [Issue #37][]. Other providers can be added in the future as Envoy Gateway use cases are better understood. A +provider is configured at start up through Envoy Gateway's [static configuration](#static-configuration). #### Kubernetes Provider @@ -88,6 +88,8 @@ Representation (IR). It is responsible for: * Translating infrastructure-specific resources/fields from the Resource Watcher to the Infra IR. * Translating proxy configuration resources/fields from the Resource Watcher to the xDS IR. +__Note:__ The Resource Translator is implemented as the `Translator` API type in the `gatewayapi` package. + ### Intermediate Representation (IR) The Intermediate Representation defines internal data models that external resources are translated into. This allows @@ -103,7 +105,7 @@ The xDS Translator translates the xDS IR into xDS Resources that are consumed by ### xDS Server -The xDS Server is a xDS gRPC Server based on [Go Control Plane][go_cp]. Go Control Plane implements the xDS Server +The xDS Server is a xDS gRPC Server based on [Go Control Plane][go_cp]. Go Control Plane implements the Delta xDS Server Protocol and is responsible for using xDS to configure the data plane. ### Infra Manager @@ -121,32 +123,34 @@ The Infra Manager consumes the Infra IR as input to manage the data plane infras ## Design Decisions -* Envoy Gateway will consume one [GatewayClass][gc] by comparing its configured controller name with +* Envoy Gateway consumes one [GatewayClass][gc] by comparing its configured controller name with `spec.controllerName` of a GatewayClass. If multiple GatewayClasses exist with the same `spec.controllerName`, Envoy - Gateway will follow Gateway API [guidelines][gwapi_conflicts] to resolve the conflict. - `gatewayclass.spec.parametersRef` refers to a custom resource for configuring the managed proxy infrastructure. If - unspecified, default configuration parameters are used for the managed proxy infrastructure. -* Envoy Gateway will manage [Gateways][gw] that reference its GatewayClass. - * The first Gateway causes Envoy Gateway to provision the managed Envoy proxy infrastructure. - * Envoy Gateway will merge multiple Gateways that match its GatewayClass and will follow Gateway API - [guidelines][gwapi_conflicts] to resolve any conflicts. - * A Gateway `listener` corresponds to a proxy [Listener][listener]. -* An [HTTPRoute][hroute] resource corresponds to a proxy [Route][route]. - * Each [backendRef][be_ref] corresponds to a proxy [Cluster][cluster]. -* The goal is to make the Infra Manager & Translator components extensible in the future. For now, extensibility can be - achieved using xDS support that Envoy Gateway will provide. + Gateway follows Gateway API [guidelines][gwapi_conflicts] to resolve the conflict. + `gatewayclass.spec.parametersRef` refers to the `EnvoyProxy` custom resource for configuring the managed proxy + infrastructure. If unspecified, default configuration parameters are used for the managed proxy infrastructure. +* Envoy Gateway manages [Gateways][gw] that reference its GatewayClass. + * A Gateway resource causes Envoy Gateway to provision managed Envoy proxy infrastructure. + * Envoy Gateway groups Listeners by Port and collapses each group of Listeners into a single Listener if the Listeners + in the group are compatible. Envoy Gateway considers Listeners to be compatible if all the following conditions are + met: + * Either each Listener within the group specifies the “HTTP” Protocol or each Listener within the group specifies + either the “HTTPS” or “TLS” Protocol. + * Each Listener within the group specifies a unique "Hostname". + * As a special case, one Listener within a group may omit "Hostname", in which case this Listener matches when no + other Listener matches. + * Envoy Gateway does __not__ merge listeners across multiple Gateways. +* Envoy Gateway follows Gateway API [guidelines][gwapi_conflicts] to resolve any conflicts. + * A Gateway `listener` corresponds to an Envoy proxy [Listener][listener]. +* An [HTTPRoute][hroute] resource corresponds to an Envoy proxy [Route][route]. + * Each [backendRef][be_ref] corresponds to an Envoy proxy [Cluster][cluster]. +* The goal is to make Envoy Gateway components extensible in the future. See the [roadmap][] for additional details. The draft for this document is [here][draft_design]. -## Caveats - -* The custom resource used to configure the data plane infrastructure is TBD. Track [issue 95][issue_95] for the latest - updates. -* Envoy Gateway's static configuration spec is currently undefined. Track [issue 95][issue_95] for the latest updates. - [gw_api]: https://gateway-api.sigs.k8s.io [gc]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#gatewayclass [gw]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#gateway +[gw_spec]: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io%2fv1beta1 [hroute]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#httproute [troute]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#tlsroute [go_cp]: https://github.com/envoyproxy/go-control-plane @@ -165,3 +169,5 @@ The draft for this document is [here][draft_design]. [svc]: https://kubernetes.io/docs/concepts/services-networking/service/ [issue_95]: https://github.com/envoyproxy/gateway/pull/95 [ wcd ]: ./watching.md +[Issue #37]: https://github.com/envoyproxy/gateway/issues/37 +[roadmap]: roadmap.md diff --git a/docs/design/watching.md b/docs/design/watching.md index 35b45abe622..b8477a30e2d 100644 --- a/docs/design/watching.md +++ b/docs/design/watching.md @@ -1,6 +1,6 @@ # Watching Components Design -Envoy Gateway is made up of several components that communicate in-process. Some of them (namely providers) watch +Envoy Gateway is made up of several components that communicate in-process. Some of them (namely Providers) watch external resources, and "publish" what they see for other components to consume; others watch what another publishes and act on it (such as the resource translator watches what the providers publish, and then publishes its own results that are watched by another component). Some of these internally published results are consumed by multiple components. From f88877d4b234cf70a1e47a54e239de942a21da88 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 28 Oct 2022 16:43:17 -0600 Subject: [PATCH 078/113] internal/gatewayapi: support HTTP method matching (#658) Closes #655. Signed-off-by: Steve Kriss --- ...single-rule-with-http-method-match.in.yaml | 31 +++++++ ...ingle-rule-with-http-method-match.out.yaml | 87 +++++++++++++++++++ internal/gatewayapi/translator.go | 7 ++ 3 files changed, 125 insertions(+) create mode 100644 internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.out.yaml diff --git a/internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.in.yaml b/internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.in.yaml new file mode 100644 index 00000000000..3d372e82c8d --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.in.yaml @@ -0,0 +1,31 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - method: POST + backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.out.yaml b/internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.out.yaml new file mode 100644 index 00000000000..d98b7037756 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-single-rule-with-http-method-match.out.yaml @@ -0,0 +1,87 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - method: POST + backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + http: + - name: envoy-gateway-gateway-1-http + address: 0.0.0.0 + port: 10080 + hostnames: + - "*" + routes: + - name: default-httproute-1-rule-0-match-0-* + headerMatches: + - name: ":method" + exact: POST + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:translator-tests + listeners: + - address: "" + ports: + - name: http + protocol: "HTTP" + servicePort: 80 + containerPort: 10080 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index a01bf1ba794..66f1f55b28c 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -1095,6 +1095,13 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } } + if match.Method != nil { + irRoute.HeaderMatches = append(irRoute.HeaderMatches, &ir.StringMatch{ + Name: ":method", + Exact: StringPtr(string(*match.Method)), + }) + } + // Add the redirect filter or direct response that were created earlier to all the irRoutes if redirectResponse != nil { irRoute.Redirect = redirectResponse From 592863ad53b097eeca602180d1394d7c4ef41522 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Fri, 28 Oct 2022 23:18:05 -0700 Subject: [PATCH 079/113] fix version to v0.2.0 in tls passthrough docs (#660) Signed-off-by: Arko Dasgupta --- docs/user/tls-passthrough.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/tls-passthrough.md b/docs/user/tls-passthrough.md index abf015006db..80968169e13 100644 --- a/docs/user/tls-passthrough.md +++ b/docs/user/tls-passthrough.md @@ -45,7 +45,7 @@ kubectl create secret tls server-certs --key=passthrough.example.com.key --cert= Deploy TLS Passthrough application Deployment, Service and TLSRoute: ```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0-rc2/examples/kubernetes/tls-passthrough.yaml +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/v0.2.0/examples/kubernetes/tls-passthrough.yaml ``` Patch the Gateway from the Quickstart guide to include a TLS listener that listens on port `6443` and is configured for TLS mode Passthrough: From 148e3dde4a035fd738a6d0a4913fa5f9cbfec5bb Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 1 Nov 2022 08:37:06 -0700 Subject: [PATCH 080/113] xds: remove total_weights from weighted cluster (#667) * xds: remove total_weights from weighted cluster Signed-off-by: Lizan Zhou --- internal/xds/translator/route.go | 4 +--- .../xds-ir/http-route-weighted-invalid-backend.routes.yaml | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index ac4178e7b49..6ad00760bcf 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -146,7 +146,6 @@ func buildXdsRouteAction(routeName string) *route.RouteAction { } func buildXdsWeightedRouteAction(httpRoute *ir.HTTPRoute) *route.RouteAction { - totalWeight := httpRoute.BackendWeights.Valid + httpRoute.BackendWeights.Invalid clusters := []*route.WeightedCluster_ClusterWeight{ { Name: "invalid-backend-cluster", @@ -162,8 +161,7 @@ func buildXdsWeightedRouteAction(httpRoute *ir.HTTPRoute) *route.RouteAction { ClusterNotFoundResponseCode: route.RouteAction_INTERNAL_SERVER_ERROR, ClusterSpecifier: &route.RouteAction_WeightedClusters{ WeightedClusters: &route.WeightedCluster{ - TotalWeight: &wrapperspb.UInt32Value{Value: totalWeight}, - Clusters: clusters, + Clusters: clusters, }, }, } diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml index d8966acfc10..789931e4411 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml @@ -14,4 +14,3 @@ weight: 1 - name: first-route weight: 1 - totalWeight: 2 From 7785aea4b2468d3858b2e86157903f96ce72d296 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Tue, 1 Nov 2022 09:37:48 -0600 Subject: [PATCH 081/113] update Envoy to v1.24 (#654) Signed-off-by: Steve Kriss --- docs/dev/README.md | 2 +- internal/ir/infra.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/dev/README.md b/docs/dev/README.md index f9b76efd50e..e64c2718ced 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -136,4 +136,4 @@ There are many other endpoints on the [Envoy admin interface][] that may be help [Kubernetes support]: https://docs.docker.com/desktop/kubernetes/ [gateway-dev]: https://hub.docker.com/r/envoyproxy/gateway-dev/tags [mac_connect]: https://github.com/chipmk/docker-mac-net-connect -[Envoy admin interface]: https://www.envoyproxy.io/docs/envoy/v1.23.0/operations/admin#operations-admin-interface +[Envoy admin interface]: https://www.envoyproxy.io/docs/envoy/latest/operations/admin#operations-admin-interface diff --git a/internal/ir/infra.go b/internal/ir/infra.go index 28021eb50bd..c898d9f881c 100644 --- a/internal/ir/infra.go +++ b/internal/ir/infra.go @@ -16,7 +16,7 @@ import ( const ( DefaultProxyName = "default" - DefaultProxyImage = "envoyproxy/envoy:v1.23-latest" + DefaultProxyImage = "envoyproxy/envoy:v1.24-latest" ) // Infra defines managed infrastructure. @@ -36,7 +36,7 @@ type ProxyInfra struct { // Config defines user-facing configuration of the managed proxy infrastructure. Config *v1alpha1.EnvoyProxy // Image is the container image used for the managed proxy infrastructure. - // If unset, defaults to "envoyproxy/envoy:v1.23-latest". + // If unset, defaults to "envoyproxy/envoy:v1.24-latest". Image string // Listeners define the listeners exposed by the proxy infrastructure. Listeners []ProxyListener From 58f262291b5877f36fe8f5763c5df7daafaec11f Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Tue, 1 Nov 2022 12:29:00 -0700 Subject: [PATCH 082/113] Adds Release Details Doc (#665) Signed-off-by: danehans Signed-off-by: danehans --- docs/releases.rst | 3 ++- docs/releases/README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docs/releases/README.md diff --git a/docs/releases.rst b/docs/releases.rst index be29c9e5f9c..090c6707fd2 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,7 @@ Releases Learn more about Envoy Gateway releases. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + releases/README releases/v0.2 diff --git a/docs/releases/README.md b/docs/releases/README.md new file mode 100644 index 00000000000..93d3366efb0 --- /dev/null +++ b/docs/releases/README.md @@ -0,0 +1,41 @@ +# Release Details + +This document provides details for Envoy Gateway releases. Envoy Gateway follows the Semantic Versioning [v2.0.0 spec][] +for release versioning. Since Envoy Gateway is a new project, minor releases are the only defined releases. Envoy +Gateway maintainers will establish additional release details, e.g. patch releases, at a future date. + +## Stable Releases + +Stable releases of Envoy Gateway include: + +* Minor Releases- A new release branch and corresponding tag are created from the `main` branch. A minor release + is supported for 6 months following the release date. As the project matures, Envoy Gateway maintainers will reassess + the support timeframe. + +Minor releases happen quarterly and follow the schedule below. + +## Release Management + +Minor releases are handled by a designated Envoy Gateway maintainer. This maintainer is considered the Release Manager +for the release. The details for creating a release are outlined in the [release guide][]. The Release Manager is +responsible for coordinating the overall release. This includes identifying issues to be fixed in the release, +communications with the Envoy Gateway community, and the mechanics of the release. + +| Quarter | Release Manager | +|:-------:|:--------------------------------------------------------------:| +| 2022 Q4 | Daneyon Hansen ([danehans](https://github.com/danehans)) | +| 2023 Q1 | TBD | + +## Release Schedule + +In order to align with the Envoy Proxy [release schedule][], Envoy Gateway releases are produced on a fixed schedule +(the 22nd day of each quarter), with an acceptable delay of up to 2 weeks, and a hard deadline of 3 weeks. + +| Version | Expected | Actual | Difference | End of Life | +|:-------:|:-----------:|:-----------:|:----------:|:-----------:| +| 0.2.0 | 2022/10/22 | 2022/10/20 | -2 day | 2023/4/20 | +| 0.3.0 | 2023/01/22 | | | | + +[v2.0.0 spec]: https://semver.org/spec/v2.0.0.html +[release guide]: ../dev/releasing.md +[release schedule]: https://github.com/envoyproxy/envoy/blob/main/RELEASES.md#major-release-schedule From 90073beed844c6218a271dcb3eb02bcde7447973 Mon Sep 17 00:00:00 2001 From: hexiaodai <40925990+hexiaodai@users.noreply.github.com> Date: Thu, 3 Nov 2022 01:41:32 +0800 Subject: [PATCH 083/113] optimized generate certificate unit test (#678) Signed-off-by: hexiaodai --- internal/crypto/certgen_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/crypto/certgen_test.go b/internal/crypto/certgen_test.go index 9adbcd752f5..db1722d7891 100644 --- a/internal/crypto/certgen_test.go +++ b/internal/crypto/certgen_test.go @@ -51,8 +51,8 @@ func TestGenerateCerts(t *testing.T) { run(t, "no configuration - use defaults", testcase{ certConfig: &Configuration{}, - wantEnvoyGatewayDNSName: "envoy-gateway", - wantEnvoyDNSName: "*.envoy-gateway-system", + wantEnvoyGatewayDNSName: DefaultEnvoyGatewayDNSPrefix, + wantEnvoyDNSName: fmt.Sprintf("*.%s", DefaultNamespace), }) } From 58aecd9659f7a7434628629fce3a638ea50d0752 Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Thu, 3 Nov 2022 01:45:22 +0800 Subject: [PATCH 084/113] support regex on http route matches (#676) Signed-off-by: zhaohuabing --- ...th-single-rule-with-multiple-rules.in.yaml | 63 ++++++++ ...h-single-rule-with-multiple-rules.out.yaml | 144 ++++++++++++++++++ internal/gatewayapi/translator.go | 20 ++- 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.out.yaml diff --git a/internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.in.yaml b/internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.in.yaml new file mode 100644 index 00000000000..9fd68520b16 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.in.yaml @@ -0,0 +1,63 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + type: Exact + value: "/exact" + headers: + - name: Header-1 + type: Exact + value: "exact" + queryParams: + - name: QueryParam-1 + type: Exact + value: "exact" + backendRefs: + - name: service-1 + port: 8080 + - matches: + - path: + type: PathPrefix + value: "/prefix" + backendRefs: + - name: service-2 + port: 8080 + - matches: + - path: + type: RegularExpression + value: "*regex*" + headers: + - name: Header-1 + type: RegularExpression + value: "*regex*" + queryParams: + - name: QueryParam-1 + type: RegularExpression + value: "*regex*" + backendRefs: + - name: service-3 + port: 8080 diff --git a/internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.out.yaml b/internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.out.yaml new file mode 100644 index 00000000000..67e4a9b5034 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-single-rule-with-multiple-rules.out.yaml @@ -0,0 +1,144 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1beta1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + type: Exact + value: "/exact" + headers: + - name: Header-1 + type: Exact + value: "exact" + queryParams: + - name: QueryParam-1 + type: Exact + value: "exact" + backendRefs: + - name: service-1 + port: 8080 + - matches: + - path: + type: PathPrefix + value: "/prefix" + backendRefs: + - name: service-2 + port: 8080 + - matches: + - path: + type: RegularExpression + value: "*regex*" + headers: + - name: Header-1 + type: RegularExpression + value: "*regex*" + queryParams: + - name: QueryParam-1 + type: RegularExpression + value: "*regex*" + backendRefs: + - name: service-3 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + http: + - name: envoy-gateway-gateway-1-http + address: 0.0.0.0 + port: 10080 + hostnames: + - "*" + routes: + - name: default-httproute-1-rule-2-match-0-* + pathMatch: + safeRegex: "*regex*" + headerMatches: + - name: "Header-1" + safeRegex: "*regex*" + queryParamMatches: + - name: "QueryParam-1" + safeRegex: "*regex*" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: default-httproute-1-rule-1-match-0-* + pathMatch: + prefix: "/prefix" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: default-httproute-1-rule-0-match-0-* + pathMatch: + exact: "/exact" + headerMatches: + - name: "Header-1" + exact: "exact" + queryParamMatches: + - name: "QueryParam-1" + exact: "exact" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:translator-tests + listeners: + - address: "" + ports: + - name: http + protocol: "HTTP" + servicePort: 80 + containerPort: 10080 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 66f1f55b28c..2f7551cd496 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -1076,22 +1076,38 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways irRoute.PathMatch = &ir.StringMatch{ Exact: match.Path.Value, } + case v1beta1.PathMatchRegularExpression: + irRoute.PathMatch = &ir.StringMatch{ + SafeRegex: match.Path.Value, + } } } for _, headerMatch := range match.Headers { - if HeaderMatchTypeDerefOr(headerMatch.Type, v1beta1.HeaderMatchExact) == v1beta1.HeaderMatchExact { + switch HeaderMatchTypeDerefOr(headerMatch.Type, v1beta1.HeaderMatchExact) { + case v1beta1.HeaderMatchExact: irRoute.HeaderMatches = append(irRoute.HeaderMatches, &ir.StringMatch{ Name: string(headerMatch.Name), Exact: StringPtr(headerMatch.Value), }) + case v1beta1.HeaderMatchRegularExpression: + irRoute.HeaderMatches = append(irRoute.HeaderMatches, &ir.StringMatch{ + Name: string(headerMatch.Name), + SafeRegex: StringPtr(headerMatch.Value), + }) } } for _, queryParamMatch := range match.QueryParams { - if QueryParamMatchTypeDerefOr(queryParamMatch.Type, v1beta1.QueryParamMatchExact) == v1beta1.QueryParamMatchExact { + switch QueryParamMatchTypeDerefOr(queryParamMatch.Type, v1beta1.QueryParamMatchExact) { + case v1beta1.QueryParamMatchExact: irRoute.QueryParamMatches = append(irRoute.QueryParamMatches, &ir.StringMatch{ Name: queryParamMatch.Name, Exact: StringPtr(queryParamMatch.Value), }) + case v1beta1.QueryParamMatchRegularExpression: + irRoute.QueryParamMatches = append(irRoute.QueryParamMatches, &ir.StringMatch{ + Name: queryParamMatch.Name, + SafeRegex: StringPtr(queryParamMatch.Value), + }) } } From c611cdc6652e6159df3df3584619c9ce799d051f Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 2 Nov 2022 14:13:35 -0700 Subject: [PATCH 085/113] Fix hostname parsing for TLSRoutes (#662) * Fix hostname parsing for TLSRoutes * use the result from `computeHosts` which returns the list of intersecting hostnames between the listener and the route to populate the SNIs field within the Xds IR TCP Listener Fixes: https://github.com/envoyproxy/gateway/issues/661 Signed-off-by: Arko Dasgupta --- .../tlsroute-with-empty-hostname.in.yaml | 41 +++++++++ .../tlsroute-with-empty-hostname.out.yaml | 83 ++++++++++++++++++ ...route-with-empty-listener-hostname.in.yaml | 43 ++++++++++ ...oute-with-empty-listener-hostname.out.yaml | 85 +++++++++++++++++++ internal/gatewayapi/translator.go | 16 ++-- 5 files changed, 257 insertions(+), 11 deletions(-) create mode 100644 internal/gatewayapi/testdata/tlsroute-with-empty-hostname.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml diff --git a/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.in.yaml b/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.in.yaml new file mode 100644 index 00000000000..071fa048f72 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.in.yaml @@ -0,0 +1,41 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 +services: +- apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-1 + spec: + clusterIP: 7.7.7.7 + ports: + - port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml new file mode 100644 index 00000000000..0b5afbd16e1 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml @@ -0,0 +1,83 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + tcp: + - name: envoy-gateway-gateway-1-tls + address: 0.0.0.0 + port: 10091 + tls: + snis: + - "*" + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:translator-tests + listeners: + - address: "" + ports: + - name: tls + protocol: "TLS" + servicePort: 91 + containerPort: 10091 diff --git a/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.in.yaml b/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.in.yaml new file mode 100644 index 00000000000..f5c8e1349a6 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.in.yaml @@ -0,0 +1,43 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + hostnames: + - "foo.com" + rules: + - backendRefs: + - name: service-1 + port: 8080 +services: +- apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-1 + spec: + clusterIP: 7.7.7.7 + ports: + - port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml new file mode 100644 index 00000000000..eff4d035480 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml @@ -0,0 +1,85 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 1 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + hostnames: + - foo.com + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + tcp: + - name: envoy-gateway-gateway-1-tls + address: 0.0.0.0 + port: 10091 + tls: + snis: + - foo.com + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:translator-tests + listeners: + - address: "" + ports: + - name: tls + protocol: "TLS" + servicePort: 91 + containerPort: 10091 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 2f7551cd496..b7011896931 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -572,20 +572,11 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap Address: "0.0.0.0", Port: uint32(containerPort), TLS: &ir.TLSInspectorConfig{ + // Since we need to first compute intersecting hostnames between the + // listener and TLS Route, populate this field after processing TLS Routes. SNIs: []string{}, }, } - if listener.Hostname == nil || *listener.Hostname == "" { - listener.SetCondition( - v1beta1.ListenerConditionReady, - metav1.ConditionFalse, - v1beta1.ListenerReasonInvalid, - "Listener is invalid, see other Conditions for details.", - ) - } - if listener.Hostname != nil && *listener.Hostname != "" { - irListener.TLS.SNIs = append(irListener.TLS.SNIs, string(*listener.Hostname)) - } gwXdsIR.TCP = append(gwXdsIR.TCP, irListener) } @@ -1390,13 +1381,16 @@ func (t *Translator) ProcessTLSRoutes(tlsRoutes []*v1alpha2.TLSRoute, gateways [ if len(hosts) == 0 { continue } + hasHostnameIntersection = true irKey := irStringKey(listener.gateway) irListener := xdsIR[irKey].GetTCPListener(irListenerName(listener)) if irListener != nil { irListener.Destinations = routeDestinations + irListener.TLS.SNIs = hosts } + // Theoretically there should only be one parent ref per // Route that attaches to a given Listener, so fine to just increment here, but we // might want to check to ensure we're not double-counting. From 0646f946ebb854adbd71b5499704e9bf9c50f1b4 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 2 Nov 2022 15:20:27 -0600 Subject: [PATCH 086/113] update codecov action to v3 (#679) Addresses deprecations, possibly helps with the upload flake. Signed-off-by: Steve Kriss --- .github/workflows/build_and_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index ac7d522cb01..9ece87ab811 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -46,7 +46,7 @@ jobs: - name: Run Coverage Tests run: make go.test.coverage - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: true files: ./coverage.xml From 10eb279ab8bb12a0e7244ae822ff980a92cc2be1 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Thu, 3 Nov 2022 09:48:44 +0800 Subject: [PATCH 087/113] feat: support versioned docs (#639) * feat: support versioned docs Signed-off-by: bitliu --- .github/workflows/build_and_test.yaml | 2 +- .github/workflows/docs.yaml | 8 +- .gitignore | 3 +- VERSION | 1 + docs/dev/DOCS.md | 41 -- docs/index.html | 5 + docs/{ => latest}/about_docs.rst | 0 docs/{ => latest}/conf.py | 22 +- docs/{ => latest}/design/config-api.md | 0 .../design/gatewayapi-translator.md | 1 - docs/{ => latest}/design/roadmap.md | 0 docs/{ => latest}/design/system-design.md | 2 - docs/{ => latest}/design/watching.md | 0 docs/{ => latest}/design_docs.rst | 0 docs/{ => latest}/dev/CODE_OF_CONDUCT.md | 0 docs/{ => latest}/dev/CONTRIBUTING.md | 0 docs/latest/dev/DOCS.md | 63 ++++ docs/{ => latest}/dev/README.md | 0 docs/{ => latest}/dev/releasing.md | 73 ++-- docs/{ => latest}/dev_docs.rst | 1 + docs/{ => latest}/get_involved.rst | 0 docs/{ => latest}/images/architecture.png | Bin docs/{ => latest}/index.rst | 4 +- docs/{ => latest}/intro/compatibility.rst | 0 docs/{ => latest}/intro/index.rst | 0 docs/{ => latest}/releases.rst | 0 docs/{ => latest}/releases/README.md | 0 docs/{ => latest}/releases/v0.2.md | 0 docs/{ => latest}/roadmap.rst | 0 docs/{ => latest}/user/http-redirect.md | 0 .../{ => latest}/user/http-request-headers.md | 0 docs/latest/user/http-routing.md | 115 ++++++ .../user/http-traffic-splitting.md | 0 docs/latest/user/quickstart.md | 81 ++++ docs/{ => latest}/user/secure-gateways.md | 0 docs/latest/user/tls-passthrough.md | 117 ++++++ docs/{ => latest}/user_docs.rst | 0 docs/v0.2.0/about_docs.rst | 9 + docs/v0.2.0/conf.py | 40 ++ docs/v0.2.0/design/config-api.md | 350 ++++++++++++++++++ docs/v0.2.0/design/gatewayapi-translator.md | 250 +++++++++++++ docs/v0.2.0/design/roadmap.md | 60 +++ docs/v0.2.0/design/system-design.md | 171 +++++++++ docs/v0.2.0/design/watching.md | 117 ++++++ docs/v0.2.0/design_docs.rst | 12 + docs/v0.2.0/dev/CODE_OF_CONDUCT.md | 3 + docs/v0.2.0/dev/CONTRIBUTING.md | 183 +++++++++ docs/v0.2.0/dev/DOCS.md | 63 ++++ docs/v0.2.0/dev/README.md | 139 +++++++ docs/v0.2.0/dev/releasing.md | 144 +++++++ docs/v0.2.0/dev_docs.rst | 13 + docs/v0.2.0/get_involved.rst | 9 + docs/v0.2.0/images/architecture.png | Bin 0 -> 449265 bytes docs/v0.2.0/index.rst | 29 ++ docs/v0.2.0/intro/compatibility.rst | 19 + docs/v0.2.0/intro/index.rst | 15 + docs/v0.2.0/releases.rst | 10 + docs/v0.2.0/releases/README.md | 41 ++ docs/v0.2.0/releases/v0.2.md | 50 +++ docs/v0.2.0/roadmap.rst | 9 + docs/v0.2.0/user/http-redirect.md | 123 ++++++ docs/v0.2.0/user/http-request-headers.md | 308 +++++++++++++++ docs/{ => v0.2.0}/user/http-routing.md | 0 docs/v0.2.0/user/http-traffic-splitting.md | 294 +++++++++++++++ docs/{ => v0.2.0}/user/quickstart.md | 0 docs/v0.2.0/user/secure-gateways.md | 260 +++++++++++++ docs/{ => v0.2.0}/user/tls-passthrough.md | 0 docs/v0.2.0/user_docs.rst | 15 + tools/make/common.mk | 2 + tools/make/docs.mk | 47 ++- tools/make/kube.mk | 7 - 71 files changed, 3202 insertions(+), 129 deletions(-) create mode 100644 VERSION delete mode 100644 docs/dev/DOCS.md create mode 100644 docs/index.html rename docs/{ => latest}/about_docs.rst (100%) rename docs/{ => latest}/conf.py (74%) rename docs/{ => latest}/design/config-api.md (100%) rename docs/{ => latest}/design/gatewayapi-translator.md (99%) rename docs/{ => latest}/design/roadmap.md (100%) rename docs/{ => latest}/design/system-design.md (98%) rename docs/{ => latest}/design/watching.md (100%) rename docs/{ => latest}/design_docs.rst (100%) rename docs/{ => latest}/dev/CODE_OF_CONDUCT.md (100%) rename docs/{ => latest}/dev/CONTRIBUTING.md (100%) create mode 100644 docs/latest/dev/DOCS.md rename docs/{ => latest}/dev/README.md (100%) rename docs/{ => latest}/dev/releasing.md (59%) rename docs/{ => latest}/dev_docs.rst (94%) rename docs/{ => latest}/get_involved.rst (100%) rename docs/{ => latest}/images/architecture.png (100%) rename docs/{ => latest}/index.rst (87%) rename docs/{ => latest}/intro/compatibility.rst (100%) rename docs/{ => latest}/intro/index.rst (100%) rename docs/{ => latest}/releases.rst (100%) rename docs/{ => latest}/releases/README.md (100%) rename docs/{ => latest}/releases/v0.2.md (100%) rename docs/{ => latest}/roadmap.rst (100%) rename docs/{ => latest}/user/http-redirect.md (100%) rename docs/{ => latest}/user/http-request-headers.md (100%) create mode 100644 docs/latest/user/http-routing.md rename docs/{ => latest}/user/http-traffic-splitting.md (100%) create mode 100644 docs/latest/user/quickstart.md rename docs/{ => latest}/user/secure-gateways.md (100%) create mode 100644 docs/latest/user/tls-passthrough.md rename docs/{ => latest}/user_docs.rst (100%) create mode 100644 docs/v0.2.0/about_docs.rst create mode 100644 docs/v0.2.0/conf.py create mode 100644 docs/v0.2.0/design/config-api.md create mode 100644 docs/v0.2.0/design/gatewayapi-translator.md create mode 100644 docs/v0.2.0/design/roadmap.md create mode 100644 docs/v0.2.0/design/system-design.md create mode 100644 docs/v0.2.0/design/watching.md create mode 100644 docs/v0.2.0/design_docs.rst create mode 100644 docs/v0.2.0/dev/CODE_OF_CONDUCT.md create mode 100644 docs/v0.2.0/dev/CONTRIBUTING.md create mode 100644 docs/v0.2.0/dev/DOCS.md create mode 100644 docs/v0.2.0/dev/README.md create mode 100644 docs/v0.2.0/dev/releasing.md create mode 100644 docs/v0.2.0/dev_docs.rst create mode 100644 docs/v0.2.0/get_involved.rst create mode 100644 docs/v0.2.0/images/architecture.png create mode 100644 docs/v0.2.0/index.rst create mode 100644 docs/v0.2.0/intro/compatibility.rst create mode 100644 docs/v0.2.0/intro/index.rst create mode 100644 docs/v0.2.0/releases.rst create mode 100644 docs/v0.2.0/releases/README.md create mode 100644 docs/v0.2.0/releases/v0.2.md create mode 100644 docs/v0.2.0/roadmap.rst create mode 100644 docs/v0.2.0/user/http-redirect.md create mode 100644 docs/v0.2.0/user/http-request-headers.md rename docs/{ => v0.2.0}/user/http-routing.md (100%) create mode 100644 docs/v0.2.0/user/http-traffic-splitting.md rename docs/{ => v0.2.0}/user/quickstart.md (100%) create mode 100644 docs/v0.2.0/user/secure-gateways.md rename docs/{ => v0.2.0}/user/tls-passthrough.md (100%) create mode 100644 docs/v0.2.0/user_docs.rst diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 9ece87ab811..03d062b2ea1 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -37,7 +37,7 @@ jobs: build-and-test: runs-on: ubuntu-latest - needs: [lint, gen-check] + needs: [lint, gen-check, license-check] steps: - uses: actions/checkout@v3 - uses: ./tools/github-actions/setup-deps diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f8b8fe3696b..bbc8d57b012 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,15 +3,11 @@ on: push: branches: - "main" - # Uncomment here until https://github.com/envoyproxy/gateway/issues/632 has been fixed. - # - "release/v*" paths-ignore: - "**/*.png" pull_request: branches: - "main" - # Uncomment here until https://github.com/envoyproxy/gateway/issues/632 has been fixed. - # - "release/v*" paths-ignore: - "**/*.png" @@ -36,8 +32,8 @@ jobs: - uses: actions/checkout@v3 - uses: ./tools/github-actions/setup-deps - # docs - - run: make docs + - name: Generate EG Pages + run: make docs # Upload docs for GitHub Pages - name: Upload GitHub Pages artifact diff --git a/.gitignore b/.gitignore index 02df3075bd3..24c1a01fee3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store bin/ -/docs/html + +docs/html # Intellij *.iml diff --git a/VERSION b/VERSION new file mode 100644 index 00000000000..1474d00f01c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.2.0 diff --git a/docs/dev/DOCS.md b/docs/dev/DOCS.md deleted file mode 100644 index 643636121bd..00000000000 --- a/docs/dev/DOCS.md +++ /dev/null @@ -1,41 +0,0 @@ -# Working on the Envoy Gateway Docs - -The documentation for the Envoy Gateway lives in the `docs/` directory. Any -individual document can be written using either [reStructuredText] or [Markdown], -you can choose the format that you're most comfortable with when working on the -documentation. - -## Documentation Structure - -The root of the site is in `docs/index.rst`. This is probably where to start -if you're trying to understand how things fit together. - -It's important to note that a given document _must_ have a reference in some -`.. toctree::` section for the document to be reachable. Not everything needs -to be in `docs/index.rst`'s `toctree` though. - -## Documentation Workflow - -To work with the docs, just edit reStructuredText or Markdown files in `docs`, -then run - -```bash -make docs -``` - -This will create `docs/html` with the built HTML pages. You can view the docs -either simply by pointing a web browser at the `file://` path to your -`docs/html`, or by firing up a static webserver from that directory, e.g. - -``` -cd docs/html ; python3 -m http.server -``` - -## Publishing Docs - -Whenever docs are pushed to `main`, CI will publish the built docs to GitHub -Pages. For more details, see `.github/workflows/docs.yaml`. - -[reStructuredText]: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html -[Markdown]: https://daringfireball.net/projects/markdown/syntax - diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 00000000000..28a7e5e4af7 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,5 @@ + + + + + diff --git a/docs/about_docs.rst b/docs/latest/about_docs.rst similarity index 100% rename from docs/about_docs.rst rename to docs/latest/about_docs.rst diff --git a/docs/conf.py b/docs/latest/conf.py similarity index 74% rename from docs/conf.py rename to docs/latest/conf.py index 8067a182843..c90c4a49fef 100644 --- a/docs/conf.py +++ b/docs/latest/conf.py @@ -16,26 +16,14 @@ master_doc = 'index' # General information about the project. +version = os.environ["BUILD_VERSION"] +envoyVersion = os.environ["ENVOY_VERSION"] +gatewayAPIVersion = os.environ["GATEWAYAPI_VERSION"] -fullversion = os.environ["BUILD_VERSION"] -release = fullversion - -version = fullversion - -m = re.match(r"^(v\d+\.\d+\.\d+)(-rc\d+)", version) - -if m: - version = "".join(m.groups()) - -release = version - -project = f'Envoy Gateway {version}' +project = 'Envoy Gateway' author = 'Envoy Gateway Project Authors' -copyright = 'Envoy Gateway Project Authors | GitHub | ' + fullversion - -envoyVersion = os.environ["ENVOY_VERSION"] -gatewayAPIVersion = os.environ["GATEWAYAPI_VERSION"] +copyright = 'Envoy Gateway Project Authors | GitHub | Latest Docs' source_suffix = { '.rst': 'restructuredtext', diff --git a/docs/design/config-api.md b/docs/latest/design/config-api.md similarity index 100% rename from docs/design/config-api.md rename to docs/latest/design/config-api.md diff --git a/docs/design/gatewayapi-translator.md b/docs/latest/design/gatewayapi-translator.md similarity index 99% rename from docs/design/gatewayapi-translator.md rename to docs/latest/design/gatewayapi-translator.md index cd09a8b6c73..1480c5f4257 100644 --- a/docs/design/gatewayapi-translator.md +++ b/docs/latest/design/gatewayapi-translator.md @@ -22,7 +22,6 @@ The outputs of the Gateway API translator are: ## Listener Compatibility - Envoy Gateway follows Gateway API listener compatibility spec: > Each listener in a Gateway must have a unique combination of Hostname, Port, and Protocol. An implementation MAY group > Listeners by Port and then collapse each group of Listeners into a single Listener if the implementation determines diff --git a/docs/design/roadmap.md b/docs/latest/design/roadmap.md similarity index 100% rename from docs/design/roadmap.md rename to docs/latest/design/roadmap.md diff --git a/docs/design/system-design.md b/docs/latest/design/system-design.md similarity index 98% rename from docs/design/system-design.md rename to docs/latest/design/system-design.md index 96747c0b27a..731cb0925b0 100644 --- a/docs/design/system-design.md +++ b/docs/latest/design/system-design.md @@ -150,7 +150,6 @@ The draft for this document is [here][draft_design]. [gw_api]: https://gateway-api.sigs.k8s.io [gc]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#gatewayclass [gw]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#gateway -[gw_spec]: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io%2fv1beta1 [hroute]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#httproute [troute]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#tlsroute [go_cp]: https://github.com/envoyproxy/go-control-plane @@ -167,7 +166,6 @@ The draft for this document is [here][draft_design]. [cr]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ [be]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.BackendObjectReference [svc]: https://kubernetes.io/docs/concepts/services-networking/service/ -[issue_95]: https://github.com/envoyproxy/gateway/pull/95 [ wcd ]: ./watching.md [Issue #37]: https://github.com/envoyproxy/gateway/issues/37 [roadmap]: roadmap.md diff --git a/docs/design/watching.md b/docs/latest/design/watching.md similarity index 100% rename from docs/design/watching.md rename to docs/latest/design/watching.md diff --git a/docs/design_docs.rst b/docs/latest/design_docs.rst similarity index 100% rename from docs/design_docs.rst rename to docs/latest/design_docs.rst diff --git a/docs/dev/CODE_OF_CONDUCT.md b/docs/latest/dev/CODE_OF_CONDUCT.md similarity index 100% rename from docs/dev/CODE_OF_CONDUCT.md rename to docs/latest/dev/CODE_OF_CONDUCT.md diff --git a/docs/dev/CONTRIBUTING.md b/docs/latest/dev/CONTRIBUTING.md similarity index 100% rename from docs/dev/CONTRIBUTING.md rename to docs/latest/dev/CONTRIBUTING.md diff --git a/docs/latest/dev/DOCS.md b/docs/latest/dev/DOCS.md new file mode 100644 index 00000000000..fb49b9d55dd --- /dev/null +++ b/docs/latest/dev/DOCS.md @@ -0,0 +1,63 @@ +# Working on the Envoy Gateway Docs + +The documentation for the Envoy Gateway lives in the `docs/` directory. Any +individual document can be written using either [reStructuredText] or [Markdown], +you can choose the format that you're most comfortable with when working on the +documentation. + +## Documentation Structure + +We supported the versioned Docs now, the directory name under docs represents +the version of docs. The root of the latest site is in `docs/latest/index.rst`. +This is probably where to start if you're trying to understand how things fit together. + +Note that the new contents should be added to `docs/latest` and will be cut off at +the next release. The contents under `docs/v0.2.0` are auto-generated, +and usually do not need to make changes to them, unless if you find the current release pages have +some incorrect contents. If so, you should send a PR to update contents both of `docs/latest` +and `docs/v0.2.0`. + +It's important to note that a given document _must_ have a reference in some +`.. toctree::` section for the document to be reachable. Not everything needs +to be in `docs/index.rst`'s `toctree` though. + +You can access the website which represents the current release in default, +and you can access the website which contains the latest version changes in +[Here][latest-website] or at the footer of the pages. + +## Documentation Workflow + +To work with the docs, just edit reStructuredText or Markdown files in `docs`, +then run + +```bash +make docs +``` + +This will create `docs/html` with the built HTML pages. You can view the docs +either simply by pointing a web browser at the `file://` path to your +`docs/html`, or by firing up a static webserver from that directory, e.g. + +``` shell +make docs-serve +``` + +If you want to generate a new release version of the docs, like `v0.3.0`, then run + +```bash +make docs-release TAG=v0.3.0 +``` + +This will update the VERSION file at the project root, which records current release version, +and it will be used in the pages version context and binary version output. Also, this will generate +new dir `docs/v0.3.0`, which contains docs at v0.3.0 and updates artifact links to `v0.3.0` +in all files under `docs/v0.3.0/user`, like `quickstart.md`, `http-routing.md` and etc. + +## Publishing Docs + +Whenever docs are pushed to `main`, CI will publish the built docs to GitHub +Pages. For more details, see `.github/workflows/docs.yaml`. + +[reStructuredText]: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html +[Markdown]: https://daringfireball.net/projects/markdown/syntax +[latest-website]: https://gateway.envoyproxy.io/latest diff --git a/docs/dev/README.md b/docs/latest/dev/README.md similarity index 100% rename from docs/dev/README.md rename to docs/latest/dev/README.md diff --git a/docs/dev/releasing.md b/docs/latest/dev/releasing.md similarity index 59% rename from docs/dev/releasing.md rename to docs/latest/dev/releasing.md index 34cc1669452..a6d965be92f 100644 --- a/docs/dev/releasing.md +++ b/docs/latest/dev/releasing.md @@ -19,45 +19,31 @@ export GITHUB_REMOTE=origin ``` 1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. -2. Create a topic branch to create the release notes. Reference previous [release notes][] for additional details. -3. Sign, commit, and push your changes to your fork and submit a [Pull Request][] to merge the release notes into the - `main` branch. -4. Create a topic branch for the release announcement. Reference previous [release announcements][] for additional - details. -5. Sign, commit, and push your changes to your fork and submit a [Pull Request][] to merge the release announcement - into the `main` branch. -6. Create a topic branch and update the release tag references in the [Quickstart Guide][]. This should be the last - commit to main before cutting the release. +2. Create a topic branch to create the release notes and release docs. Reference previous [release notes][] for additional details. +3. Sign, commit, and push your changes to your fork and submit a [Pull Request][] to merge the changes listed below + into the `main` branch. Do not proceed until all your PRs have merged and the [Build and Test][build-and-test GitHub action] has completed for your final PR: - ```shell - make update-quickstart TAG=${MAJOR_VERSION}.${MINOR_VERSION}.0 - ``` - -7. Sign, commit, and push your changes to your fork. Send a PR to get your changes merged into main. Do not proceed - until all your PRs have merged and the Build and Test [release GitHub action][] has completed for your final PR. + 1. Add Release Announcement. + 2. Add Release Versioned Documents. - __Note:__ The Quickstart update should occur in the release branch after [Issue #632][] is resolved. - -8. Pull the latest changes from the `main` branch that include commits from all the above PRs: - - ```shell - git pull ${GITHUB_REMOTE} main + ``` shell + make docs-release TAG=v${MAJOR_VERSION}.${MINOR_VERSION}.0 ``` -9. Create a new release branch from `main`. The release branch should be named +4. Create a new release branch from `main`. The release branch should be named `release/v${MAJOR_VERSION}.${MINOR_VERSION}`, e.g. `release/v0.3`. ```shell git checkout -b release/v${MAJOR_VERSION}.${MINOR_VERSION} ``` -10. Push the branch to the Envoy Gateway repo. +5. Push the branch to the Envoy Gateway repo. ```shell git push ${GITHUB_REMOTE} release/v${MAJOR_VERSION}.${MINOR_VERSION} ``` -11. Tag the head of your release branch with the release tag. For example: +6. Tag the head of your release branch with the release tag. For example: ```shell git tag -a v${MAJOR_VERSION}.${MINOR_VERSION}.0 -m 'Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0 Release' @@ -65,18 +51,18 @@ export GITHUB_REMOTE=origin __Note:__ The tag version differs from the release branch by including the `.0` patch version. -12. Push the tag to the Envoy Gateway repository. +7. Push the tag to the Envoy Gateway repository. ```shell git push origin v${MAJOR_VERSION}.${MINOR_VERSION}.0 ``` -13. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. -14. Confirm that the [release workflow][] completed successfully. -15. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. -16. Confirm that the [release][] was created. -17. Confirm that the steps in the [Quickstart Guide][] work as expected. -18. [Generate][] the GitHub changelog and include the following text at the beginning of the release page: +8. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +9. Confirm that the [release workflow][] completed successfully. +10. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +11. Confirm that the [release][] was created. +12. Confirm that the steps in the [Quickstart Guide][] work as expected. +13. [Generate][] the GitHub changelog and include the following text at the beginning of the release page: ```console # Release Announcement @@ -85,7 +71,7 @@ export GITHUB_REMOTE=origin (https://gateway.envoyproxy.io/releases/v${MAJOR_VERSION}.${MINOR_VERSION}.html) to learn more about the release. ``` -19. If you find any bugs in this process, please create an issue. +If you find any bugs in this process, please create an issue. ## Creating a Release Candidate @@ -103,27 +89,27 @@ export GITHUB_REMOTE=origin ``` 1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. -5. Tag the head of the main branch with the release candidate number. +2. Tag the head of the main branch with the release candidate number. ```shell git tag -a v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} -m 'Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} Release Candidate' ``` -6. Push the tag to the Envoy Gateway repository. +3. Push the tag to the Envoy Gateway repository. ```shell git push v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} ``` -7. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. -8. Confirm that the [release workflow][] completed successfully. -9. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. -10. Confirm that the [release][] was created. -11. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test +4. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +5. Confirm that the [release workflow][] completed successfully. +6. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +7. Confirm that the [release][] was created. +8. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test the quickstart steps using the release candidate by manually updating the links. -12. [Generate][] the GitHub changelog. -13. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. -14. If you find any bugs in this process, please create an issue. +9. [Generate][] the GitHub changelog. +10. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. +11. If you find any bugs in this process, please create an issue. ## Announcing the Release @@ -150,10 +136,9 @@ It's important that the world knows about the release. Use the following steps t [release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes [Pull Request]: https://github.com/envoyproxy/gateway/pulls [Quickstart Guide]: https://github.com/envoyproxy/gateway/blob/main/docs/user/quickstart.md +[build-and-test GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/build_and_test.yaml [release GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/release.yaml [release workflow]: https://github.com/envoyproxy/gateway/actions/workflows/release.yaml [image]: https://hub.docker.com/r/envoyproxy/gateway/tags [release]: https://github.com/envoyproxy/gateway/releases [Generate]: https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes -[release announcements]: https://github.com/envoyproxy/gateway/blob/main/docs/releases/v0.2.md -[Issue #632]: https://github.com/envoyproxy/gateway/issues/632 diff --git a/docs/dev_docs.rst b/docs/latest/dev_docs.rst similarity index 94% rename from docs/dev_docs.rst rename to docs/latest/dev_docs.rst index 885b86cd18a..88a0b367f01 100644 --- a/docs/dev_docs.rst +++ b/docs/latest/dev_docs.rst @@ -9,4 +9,5 @@ Learn how to contribute to Envoy Gateway. dev/CODE_OF_CONDUCT dev/CONTRIBUTING dev/README + dev/DOCS dev/releasing diff --git a/docs/get_involved.rst b/docs/latest/get_involved.rst similarity index 100% rename from docs/get_involved.rst rename to docs/latest/get_involved.rst diff --git a/docs/images/architecture.png b/docs/latest/images/architecture.png similarity index 100% rename from docs/images/architecture.png rename to docs/latest/images/architecture.png diff --git a/docs/index.rst b/docs/latest/index.rst similarity index 87% rename from docs/index.rst rename to docs/latest/index.rst index bfcb9180980..0c054b057d0 100644 --- a/docs/index.rst +++ b/docs/latest/index.rst @@ -1,7 +1,7 @@ -Envoy Gateway +`Envoy Gateway `_ ============= -Release |version| (Envoy |envoyVersion|, Gateway API |gatewayAPIVersion|) +Release |version| .. image:: https://img.shields.io/badge/slack-join-orange.svg :target: https://envoyproxy.slack.com/archives/C03E6NHLESV diff --git a/docs/intro/compatibility.rst b/docs/latest/intro/compatibility.rst similarity index 100% rename from docs/intro/compatibility.rst rename to docs/latest/intro/compatibility.rst diff --git a/docs/intro/index.rst b/docs/latest/intro/index.rst similarity index 100% rename from docs/intro/index.rst rename to docs/latest/intro/index.rst diff --git a/docs/releases.rst b/docs/latest/releases.rst similarity index 100% rename from docs/releases.rst rename to docs/latest/releases.rst diff --git a/docs/releases/README.md b/docs/latest/releases/README.md similarity index 100% rename from docs/releases/README.md rename to docs/latest/releases/README.md diff --git a/docs/releases/v0.2.md b/docs/latest/releases/v0.2.md similarity index 100% rename from docs/releases/v0.2.md rename to docs/latest/releases/v0.2.md diff --git a/docs/roadmap.rst b/docs/latest/roadmap.rst similarity index 100% rename from docs/roadmap.rst rename to docs/latest/roadmap.rst diff --git a/docs/user/http-redirect.md b/docs/latest/user/http-redirect.md similarity index 100% rename from docs/user/http-redirect.md rename to docs/latest/user/http-redirect.md diff --git a/docs/user/http-request-headers.md b/docs/latest/user/http-request-headers.md similarity index 100% rename from docs/user/http-request-headers.md rename to docs/latest/user/http-request-headers.md diff --git a/docs/latest/user/http-routing.md b/docs/latest/user/http-routing.md new file mode 100644 index 00000000000..67137f571ac --- /dev/null +++ b/docs/latest/user/http-routing.md @@ -0,0 +1,115 @@ +# HTTP Routing + +The [HTTPRoute][] resource allows users to configure HTTP routing by matching HTTP traffic and forwarding it to +Kubernetes backends. Currently, the only supported backend supported by Envoy Gateway is a Service resource. This guide +shows how to route traffic based on host, header, and path fields and forward the traffic to different Kubernetes +Services. To learn more about HTTP routing, refer to the [Gateway API documentation][]. + +Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and then install the example +resources used for this guide. + +```shell +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/kubernetes/http-routing.yaml +``` + +The manifest installs a [GatewayClass][], [Gateway][], four Deployments, four Services, and three HTTPRoute resources. + +The GatewayClass is a cluster-scoped resource that represents a class of Gateways that can be instantiated. Envoy +Gateway is configured by default to manage GatewayClasses with +`controllerName: gateway.envoyproxy.io/gatewayclass-controller`. + +Check the status of the GatewayClass: + +```shell +kubectl get gc --selector=example=http-routing +``` + +The status should reflect "Accepted=True", indicating Envoy Gateway is managing the GatewayClass. + +A Gateway represents configuration of infrastructure. When a Gateway is created, [Envoy proxy][] infrastructure is +provisioned or configured by Envoy Gateway. The `gatewayClassName` defines the name of a GatewayClass used by this +Gateway. Check the status of the Gateway: + +```shell +kubectl get gateways --selector=example=http-routing +``` + +The status should reflect "Ready=True", indicating the Envoy proxy infrastructure has been provisioned. The status also +provides the address of the Gateway. This address is used later in the guide to test connectivity to proxied backend +services. + +The three HTTPRoute resources create routing rules on the Gateway. In order to receive traffic from a Gateway, +an HTTPRoute must be configured with `parentRefs` which reference the parent Gateway(s) that it should be attached to. +An HTTPRoute can match against a [single set of hostnames][spec]. These hostnames are matched before any other matching +within the HTTPRoute takes place. Since `example.com`, `foo.example.com`, and `bar.example.com` are separate hosts with +different routing requirements, each is deployed as its own HTTPRoute - `example-route, ``foo-route`, and `bar-route`. + +Check the status of the HTTPRoutes: + +```shell +kubectl get httproutes --selector=example=http-routing -o yaml +``` + +The status for each HTTPRoute should surface "Accepted=True" and a `parentRef` that references the example Gateway. +The `example-route` matches any traffic for "example.com" and forwards it to the "example-svc" Service. Before testing +HTTP routing to the `example-svc` backend, get the Gateway's address. + +```shell +export GATEWAY_HOST=$(kubectl get gateway/example-gateway -o jsonpath='{.status.addresses[0].value}') +``` + +Test HTTP routing to the `example-svc` backend. + +```shell +curl -vvv --header "Host: example.com" "http://${GATEWAY_HOST}/" +``` + +A `200` status code should be returned and the body should include `"pod": "example-backend-*"` indicating the traffic +was routed to the example backend service. If you change the hostname to a hostname not represented in any of the +HTTPRoutes, e.g. "www.example.com", the HTTP traffic will not be routed and a `404` should be returned. + +The `foo-route` matches any traffic for `foo.example.com` and applies its routing rules to forward the traffic to the +"foo-svc" Service. Since there is only one path prefix match for `/login`, only `foo.example.com/login/*` traffic will +be forwarded. Test HTTP routing to the `foo-svc` backend. + +```shell +curl -vvv --header "Host: foo.example.com" "http://${GATEWAY_HOST}/login" +``` + +A `200` status code should be returned and the body should include `"pod": "foo-backend-*"` indicating the traffic +was routed to the foo backend service. Traffic to any other paths that do not begin with `/login` will not be matched by +this HTTPRoute. Test this by removing `/login` from the request. + +```shell +curl -vvv --header "Host: foo.example.com" "http://${GATEWAY_HOST}/" +``` + +The HTTP traffic will not be routed and a `404` should be returned. + +Similarly, the `bar-route` HTTPRoute matches traffic for `bar.example.com`. All traffic for this hostname will be +evaluated against the routing rules. The most specific match will take precedence which means that any traffic with the +`env:canary` header will be forwarded to `bar-svc-canary` and if the header is missing or not `canary` then it'll be +forwarded to `bar-svc`. Test HTTP routing to the `bar-svc` backend. + +```shell +curl -vvv --header "Host: bar.example.com" "http://${GATEWAY_HOST}/" +``` + +A `200` status code should be returned and the body should include `"pod": "bar-backend-*"` indicating the traffic +was routed to the foo backend service. + +Test HTTP routing to the `bar-canary-svc` backend by adding the `env: canary` header to the request. + +```shell +curl -vvv --header "Host: bar.example.com" --header "env: canary" "http://${GATEWAY_HOST}/" +``` + +A `200` status code should be returned and the body should include `"pod": "bar-canary-backend-*"` indicating the +traffic was routed to the foo backend service. + +[HTTPRoute]: https://gateway-api.sigs.k8s.io/api-types/httproute/ +[Gateway API documentation]: https://gateway-api.sigs.k8s.io/ +[GatewayClass]: https://gateway-api.sigs.k8s.io/api-types/gatewayclass/ +[Gateway]: https://gateway-api.sigs.k8s.io/api-types/gateway/ +[Envoy proxy]: https://www.envoyproxy.io/ +[spec]: /references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteSpec diff --git a/docs/user/http-traffic-splitting.md b/docs/latest/user/http-traffic-splitting.md similarity index 100% rename from docs/user/http-traffic-splitting.md rename to docs/latest/user/http-traffic-splitting.md diff --git a/docs/latest/user/quickstart.md b/docs/latest/user/quickstart.md new file mode 100644 index 00000000000..993003e2419 --- /dev/null +++ b/docs/latest/user/quickstart.md @@ -0,0 +1,81 @@ +# Quickstart + +This guide will help you get started with Envoy Gateway in a few simple steps. + +## Prerequisites + +A Kubernetes cluster. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. + +## Installation + +Install the Gateway API CRDs and Envoy Gateway: + +```shell +kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/latest/install.yaml +``` + +Install the GatewayClass, Gateway, HTTPRoute and example app: + +```shell +kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/latest/quickstart.yaml +``` + +## Testing the Configuration + +Get the name of the Envoy service created the by the example Gateway: + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8888:8080 & +``` + +Curl the example app through Envoy proxy: + +```shell +curl --verbose --header "Host: www.example.com" http://localhost:8888/get +``` + +### External LoadBalancer Support + +You can also test the same functionality by sending traffic to the External IP. To get the external IP of the +Envoy service, run: + +```shell +export GATEWAY_HOST=$(kubectl get svc/${ENVOY_SERVICE} -n envoy-gateway-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +``` + +In certain environments, the load balancer may be exposed using a hostname, instead of an IP address. If so, replace +`ip` in the above command with `hostname`. + +Curl the example app through Envoy proxy: + +```shell +curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST:8080/get +``` + +## Clean-Up + +Use the steps in this section to uninstall everything from the quickstart guide. + +Delete the GatewayClass, Gateway, HTTPRoute and Example App: + +```shell +kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/latest/quickstart.yaml --ignore-not-found=true +``` + +Delete the Gateway API CRDs and Envoy Gateway: + +```shell +kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/latest/install.yaml --ignore-not-found=true +``` + +## Next Steps + +Checkout the [Developer Guide](../dev/README.md) to get involved in the project. diff --git a/docs/user/secure-gateways.md b/docs/latest/user/secure-gateways.md similarity index 100% rename from docs/user/secure-gateways.md rename to docs/latest/user/secure-gateways.md diff --git a/docs/latest/user/tls-passthrough.md b/docs/latest/user/tls-passthrough.md new file mode 100644 index 00000000000..eb830e64dd6 --- /dev/null +++ b/docs/latest/user/tls-passthrough.md @@ -0,0 +1,117 @@ +# TLS Passthrough + +This guide will walk through the steps required to configure TLS Passthrough via Envoy Gateway. Unlike configuring Secure Gateways, where the Gateway terminates the client TLS connection, TLS Passthrough allows the application itself to terminate the TLS connection, while the Gateway routes the requests to the application based on SNI headers. + +## Prerequisites + +- A Kubernetes cluster with `kubectl` context configured for the cluster. +- OpenSSL to generate TLS assets. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. + +## Installation + +Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and the example manifest. +Before proceeding, you should be able to curl the example backend using HTTP. + +## TLS Certificates + +Generate the certificates and keys used by the Service to terminate client TLS connections. +For the application, we'll deploy a sample echoserver app, with the certificates loaded in the application Pod. + +__Note:__ These certificates will not be used by the Gateway, but will remain in the application scope. + +Create a root certificate and private key to sign certificates: + +```shell +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=example Inc./CN=example.com' -keyout example.com.key -out example.com.crt +``` + +Create a certificate and a private key for `passthrough.example.com`: + +```shell +openssl req -out passthrough.example.com.csr -newkey rsa:2048 -nodes -keyout passthrough.example.com.key -subj "/CN=passthrough.example.com/O=some organization" +openssl x509 -req -sha256 -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in passthrough.example.com.csr -out passthrough.example.com.crt +``` + +Store the cert/keys in A Secret: + +```shell +kubectl create secret tls server-certs --key=passthrough.example.com.key --cert=passthrough.example.com.crt +``` + +## Deployment + +Deploy TLS Passthrough application Deployment, Service and TLSRoute: + +```shell +kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/kubernetes/tls-passthrough.yaml +``` + +Patch the Gateway from the Quickstart guide to include a TLS listener that listens on port `6443` and is configured for TLS mode Passthrough: + +```console +$ kubectl patch gateway eg --type=json --patch '[{ + "op": "add", + "path": "/spec/listeners/-", + "value": { + "name": "tls", + "protocol": "TLS", + "hostname": "passthrough.example.com", + "tls": {"mode": "Passthrough"}, + "port": 6443, + }, +}]' +``` + +## Testing + +### Clusters without External LoadBalancer Support + +Get the name of the Envoy service created the by the example Gateway: + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 6043:6443 & +``` + +Curl the example app through Envoy proxy: + +```shell +curl -v --resolve "passthrough.example.com:6043:127.0.0.1" https://passthrough.example.com:6043 \ +--cacert passthrough.example.com.crt +``` + +### Clusters with External LoadBalancer Support + +You can also test the same functionality by sending traffic to the External IP of the Gateway: + +```shell +export GATEWAY_HOST=$(kubectl get gateway/eg -o jsonpath='{.status.addresses[0].value}') +``` + +Curl the example app through the Gateway, e.g. Envoy proxy: + +```shell +curl -v -HHost:passthrough.example.com --resolve "passthrough.example.com:6443:${GATEWAY_HOST}" \ +--cacert example.com.crt https://passthrough.example.com:6443/get +``` + +## Clean-Up + +Follow the steps from the [Quickstart Guide](quickstart.md) to uninstall Envoy Gateway and the example manifest. + +Delete the Secret: + +```shell +kubectl delete secret/server-certs +``` + +## Next Steps + +Checkout the [Developer Guide](../dev/README.md) to get involved in the project. diff --git a/docs/user_docs.rst b/docs/latest/user_docs.rst similarity index 100% rename from docs/user_docs.rst rename to docs/latest/user_docs.rst diff --git a/docs/v0.2.0/about_docs.rst b/docs/v0.2.0/about_docs.rst new file mode 100644 index 00000000000..64a12d791d7 --- /dev/null +++ b/docs/v0.2.0/about_docs.rst @@ -0,0 +1,9 @@ +About the documentation +======================= + +Learn how to contribute to Envoy Gateway documentation. + +.. toctree:: + :maxdepth: 1 + + dev/DOCS diff --git a/docs/v0.2.0/conf.py b/docs/v0.2.0/conf.py new file mode 100644 index 00000000000..c90c4a49fef --- /dev/null +++ b/docs/v0.2.0/conf.py @@ -0,0 +1,40 @@ +import os +import re + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.autosectionlabel', + 'myst_parser', +] + +html_theme = 'alabaster' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +version = os.environ["BUILD_VERSION"] +envoyVersion = os.environ["ENVOY_VERSION"] +gatewayAPIVersion = os.environ["GATEWAYAPI_VERSION"] + +project = 'Envoy Gateway' +author = 'Envoy Gateway Project Authors' + +copyright = 'Envoy Gateway Project Authors | GitHub | Latest Docs' + +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + +variables_to_export = [ + "envoyVersion", + "gatewayAPIVersion", +] + +frozen_locals = dict(locals()) +rst_epilog = '\n'.join(map(lambda x: f".. |{x}| replace:: {frozen_locals[x]}", variables_to_export)) +del frozen_locals diff --git a/docs/v0.2.0/design/config-api.md b/docs/v0.2.0/design/config-api.md new file mode 100644 index 00000000000..3696860dd54 --- /dev/null +++ b/docs/v0.2.0/design/config-api.md @@ -0,0 +1,350 @@ +# Configuration API Design + +## Motivation + +[Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway. The control plane is configured +statically at startup and the data plane is configured dynamically through Kubernetes resources, primarily +[Gateway API][gw_api] objects. Refer to the Envoy Gateway [design doc][design_doc] for additional details regarding +Envoy Gateway terminology and configuration. + +## Goals + +* Define an __initial__ API to configure Envoy Gateway at startup. +* Define an __initial__ API for configuring the managed data plane, e.g. Envoy proxies. + +## Non-Goals + +* Implementation of the configuration APIs. +* Define the `status` subresource of the configuration APIs. +* Define a __complete__ set of APIs for configuring Envoy Gateway. As stated in the [Goals](#goals), this document + defines the initial configuration APIs. +* Define an API for deploying/provisioning/operating Envoy Gateway. If needed, a future Envoy Gateway operator would be + responsible for designing and implementing this type of API. +* Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. + +## Control Plane API + +The `EnvoyGateway` API defines the control plane configuration, e.g. Envoy Gateway. Key points of this API are: + +* It will define Envoy Gateway's startup configuration file. If the file does not exist, Envoy Gateway will start up + with default configuration parameters. +* EnvoyGateway inlines the `TypeMeta` API. This allows EnvoyGateway to be versioned and managed as a GroupVersionKind + scheme. +* EnvoyGateway does not contain a metadata field since it's currently represented as a static configuration file instead of + a Kubernetes resource. +* Since EnvoyGateway does not surface status, EnvoyGatewaySpec is inlined. +* If data plane static configuration is required in the future, Envoy Gateway will use a separate file for this purpose. + +The `v1alpha1` version and `config.gateway.envoyproxy.io` API group get generated: + +```go +// gateway/api/config/v1alpha1/doc.go + +// Package v1alpha1 contains API Schema definitions for the config.gateway.envoyproxy.io API group. +// +// +groupName=config.gateway.envoyproxy.io +package v1alpha1 +``` + +The initial `EnvoyGateway` API: + +```go +// gateway/api/config/v1alpha1/envoygateway.go + +package valpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EnvoyGateway is the Schema for the envoygateways API +type EnvoyGateway struct { + metav1.TypeMeta `json:",inline"` + + // EnvoyGatewaySpec defines the desired state of Envoy Gateway. + EnvoyGatewaySpec `json:",inline"` +} + +// EnvoyGatewaySpec defines the desired state of Envoy Gateway configuration. +type EnvoyGatewaySpec struct { + // Gateway defines Gateway-API specific configuration. If unset, default + // configuration parameters will apply. + // + // +optional + Gateway *Gateway `json:"gateway,omitempty"` + + // Provider defines the desired provider configuration. If unspecified, + // the Kubernetes provider is used with default parameters. + // + // +optional + Provider *Provider `json:"provider,omitempty"` +} + +// Gateway defines desired Gateway API configuration of Envoy Gateway. +type Gateway struct { + // ControllerName defines the name of the Gateway API controller. If unspecified, + // defaults to "gateway.envoyproxy.io/gatewayclass-controller". See the following + // for additional details: + // + // https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass + // + // +optional + ControllerName string `json:"controllerName,omitempty"` +} + +// Provider defines the desired configuration of a provider. +// +union +type Provider struct { + // Type is the type of provider to use. If unset, the Kubernetes provider is used. + // + // +unionDiscriminator + Type ProviderType `json:"type,omitempty"` + // Kubernetes defines the configuration of the Kubernetes provider. Kubernetes + // provides runtime configuration via the Kubernetes API. + // + // +optional + Kubernetes *KubernetesProvider `json:"kubernetes,omitempty"` + + // File defines the configuration of the File provider. File provides runtime + // configuration defined by one or more files. + // + // +optional + File *FileProvider `json:"file,omitempty"` +} + +// ProviderType defines the types of providers supported by Envoy Gateway. +type ProviderType string + +const ( + // KubernetesProviderType defines the "Kubernetes" provider. + KubernetesProviderType ProviderType = "Kubernetes" + + // FileProviderType defines the "File" provider. + FileProviderType ProviderType = "File" +) + +// KubernetesProvider defines configuration for the Kubernetes provider. +type KubernetesProvider struct { + // TODO: Add config as use cases are better understood. +} + +// FileProvider defines configuration for the File provider. +type FileProvider struct { + // TODO: Add config as use cases are better understood. +} +``` + +__Note:__ Provider-specific configuration is defined in the `{$PROVIDER_NAME}Provider` API. + +### Gateway + +Gateway defines desired configuration of [Gateway API][gw_api] controllers that reconcile and translate Gateway API +resources into the Intermediate Representation (IR). Refer to the Envoy Gateway [design doc][design_doc] for additional +details. + +### Provider + +Provider defines the desired configuration of an Envoy Gateway provider. A provider is an infrastructure component that +Envoy Gateway calls to establish its runtime configuration. Provider is a [union type][union]. Therefore, Envoy Gateway +can be configured with only one provider based on the `type` discriminator field. Refer to the Envoy Gateway +[design doc][design_doc] for additional details. + +### Control Plane Configuration + +The configuration file is defined by the EnvoyGateway API type. At startup, Envoy Gateway searches for the configuration +at "/etc/envoy-gateway/config.yaml". + +Start Envoy Gateway: + +```shell +$ ./envoy-gateway +``` + +Since the configuration file does not exist, Envoy Gateway will start with default configuration parameters. + +The Kubernetes provider can be configured explicitly using `provider.kubernetes`: + +```yaml +$ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: Kubernetes + kubernetes: {} +EOF +``` + +This configuration will cause Envoy Gateway to use the Kubernetes provider with default configuration parameters. + +The Kubernetes provider can be configured using the `provider` field. For example, the `foo` field can be set to "bar": + +```yaml +$ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: Kubernetes + kubernetes: + foo: bar +EOF +``` + +__Note:__ The Provider API from the Kubernetes package is currently undefined and `foo: bar` is provided for +illustration purposes only. + +The same API structure is followed for each supported provider. The following example causes Envoy Gateway to use the +File provider: + +```yaml +$ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: File + file: + foo: bar +EOF +``` + +__Note:__ The Provider API from the File package is currently undefined and `foo: bar` is provided for illustration +purposes only. + +Gateway API-related configuration is expressed through the `gateway` field. If unspecified, Envoy Gateway will use +default configuration parameters for `gateway`. The following example causes the [GatewayClass][gc] controller to +manage GatewayClasses with controllerName `foo` instead of the default `gateway.envoyproxy.io/gatewayclass-controller`: + +```yaml +$ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: foo +``` + +With any of the above configuration examples, Envoy Gateway can be started without any additional arguments: + +```shell +$ ./envoy-gateway +``` + +## Data Plane API + +The data plane is configured dynamically through Kubernetes resources, primarily [Gateway API][gw_api] objects. +Optionally, the data plane infrastructure can be configured by referencing a [custom resource (CR)][cr] through +`spec.parametersRef` of the managed GatewayClass. The `EnvoyProxy` API defines the data plane infrastructure +configuration and is represented as the CR referenced by the managed GatewayClass. Key points of this API are: + +* If unreferenced by `gatewayclass.spec.parametersRef`, default parameters will be used to configure the data plane + infrastructure, e.g. expose Envoy network endpoints using a LoadBalancer service. +* Envoy Gateway will follow Gateway API [recommendations][gc] regarding updates to the EnvoyProxy CR: + > It is recommended that this resource be used as a template for Gateways. This means that a Gateway is based on the + > state of the GatewayClass at the time it was created and changes to the GatewayClass or associated parameters are + > not propagated down to existing Gateways. + +The initial `EnvoyProxy` API: + +```go +// gateway/api/config/v1alpha1/envoyproxy.go + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EnvoyProxy is the Schema for the envoyproxies API. +type EnvoyProxy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EnvoyProxySpec `json:"spec,omitempty"` + Status EnvoyProxyStatus `json:"status,omitempty"` +} + +// EnvoyProxySpec defines the desired state of Envoy Proxy infrastructure +// configuration. +type EnvoyProxySpec struct { + // Undefined by this design spec. +} + +// EnvoyProxyStatus defines the observed state of EnvoyProxy. +type EnvoyProxyStatus struct { + // Undefined by this design spec. +} +``` + +The EnvoyProxySpec and EnvoyProxyStatus fields will be defined in the future as proxy infrastructure configuration use +cases are better understood. + +### Data Plane Configuration + +GatewayClass and Gateway resources define the data plane infrastructure. Note that all examples assume Envoy Gateway is +running with the Kubernetes provider. + +```yaml +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: GatewayClass +metadata: + name: example-class +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: example-gateway +spec: + gatewayClassName: example-class + listeners: + - name: http + protocol: HTTP + port: 80 +``` + +Since the GatewayClass does not define `spec.parametersRef`, the data plane is provisioned using default configuration +parameters. The Envoy proxies will be configured with a http listener and a Kubernetes LoadBalancer service listening +on port 80. + +The following example will configure the data plane to use a ClusterIP service instead of the default LoadBalancer +service: + +```yaml +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: GatewayClass +metadata: + name: example-class +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + name: example-config + group: config.gateway.envoyproxy.io + kind: EnvoyProxy +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: example-gateway +spec: + gatewayClassName: example-class + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: example-config +spec: + networkPublishing: + type: ClusterIPService +``` + +__Note:__ The NetworkPublishing API is currently undefined and is provided here for illustration purposes only. + +[issue_51]: https://github.com/envoyproxy/gateway/issues/51 +[design_doc]: https://github.com/envoyproxy/gateway/blob/main/docs/design/SYSTEM_DESIGN.md +[gw_api]: https://gateway-api.sigs.k8s.io/ +[gc]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass +[cr]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ +[union]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#unions diff --git a/docs/v0.2.0/design/gatewayapi-translator.md b/docs/v0.2.0/design/gatewayapi-translator.md new file mode 100644 index 00000000000..1480c5f4257 --- /dev/null +++ b/docs/v0.2.0/design/gatewayapi-translator.md @@ -0,0 +1,250 @@ +# Gateway API Translator Design + +The Gateway API translates external resources, e.g. GatewayClass, from the configured Provider to the Intermediate +Representation (IR). + +## Assumptions + +Initially target core conformance features only, to be followed by extended conformance features. + +## Inputs and Outputs + +The main inputs to the Gateway API translator are: + +- GatewayClass, Gateway, HTTPRoute, TLSRoute, Service, ReferenceGrant, Namespace, and Secret resources. + +__Note:__ ReferenceGrant is not fully implemented as of v0.2. + +The outputs of the Gateway API translator are: + +- Xds and Infra Internal Representations (IRs). +- Status updates for GatewayClass, Gateways, HTTPRoutes + +## Listener Compatibility + +Envoy Gateway follows Gateway API listener compatibility spec: +> Each listener in a Gateway must have a unique combination of Hostname, Port, and Protocol. An implementation MAY group +> Listeners by Port and then collapse each group of Listeners into a single Listener if the implementation determines +> that the Listeners in the group are “compatible”. + +__Note:__ Envoy Gateway does not collapse listeners across multiple Gateways. + +### Listener Compatibility Examples + +#### Example 1: Gateway with compatible Listeners (same port & protocol, different hostnames) + +```yaml +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: gateway-1 + namespace: envoy-gateway +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + hostname: *.envoygateway.io + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + hostname: whales.envoygateway.io +``` + +#### Example 2: Gateway with compatible Listeners (same port & protocol, one hostname specified, one not) + +```yaml +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: gateway-1 + namespace: envoy-gateway +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + hostname: *.envoygateway.io + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +``` + +#### Example 3: Gateway with incompatible Listeners (same port, protocol and hostname) + +```yaml +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: gateway-1 + namespace: envoy-gateway +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + hostname: whales.envoygateway.io + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + hostname: whales.envoygateway.io +``` + +#### Example 4: Gateway with incompatible Listeners (neither specify a hostname) + +```yaml +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: gateway-1 + namespace: envoy-gateway +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +``` + +## Computing Status + +Gateway API specifies a rich set of status fields & conditions for each resource. To achieve conformance, Envoy Gateway +must compute the appropriate status fields and conditions for managed resources. + +Status is computed and set for: + +- The managed GatewayClass (`gatewayclass.status.conditions`). +- Each managed Gateway, based on its Listeners' status (`gateway.status.conditions`). For the Kubernetes provider, the + Envoy Deployment and Service status are also included to calculate Gateway status. +- Listeners for each Gateway (`gateway.status.listeners`). +- The ParentRef for each Route (`route.status.parents`). + +The Gateway API translator is responsible for calculating status conditions while translating Gateway API resources to +the IR and publishing status over the [message bus][]. The Status Manager subscribes to these status messages and +updates the resource status using the configured provider. For example, the Status Manager uses a Kubernetes client to +update resource status on the Kubernetes API server. + +## Outline + +The following roughly outlines the translation process. Each step may produce (1) IR; and (2) status updates on Gateway +API resources. + +1. Process Gateway Listeners + - Validate unique hostnames, ports, and protocols. + - Validate and compute supported kinds. + - Validate allowed namespaces (validate selector if specified). + - Validate TLS fields if specified, including resolving referenced Secrets. + +2. Process HTTPRoutes + - foreach route rule: + - compute matches + - [core] path exact, path prefix + - [core] header exact + - [extended] query param exact + - [extended] HTTP method + - compute filters + - [core] request header modifier (set/add/remove) + - [core] request redirect (hostname, statuscode) + - [extended] request mirror + - compute backends + - [core] Kubernetes services + - foreach route parent ref: + - get matching listeners (check Gateway, section name, listener validation status, listener allowed routes, hostname intersection) + - foreach matching listener: + - foreach hostname intersection with route: + - add each computed route rule to host + +## Context Structs + +To help store, access and manipulate information as it's processed during the translation process, a set of context +structs are used. These structs wrap a given Gateway API type, and add additional fields and methods to support +processing. + +`GatewayContext` wraps a Gateway and provides helper methods for setting conditions, accessing Listeners, etc. + +```go +type GatewayContext struct { + // The managed Gateway + *v1beta1.Gateway + + // A list of Gateway ListenerContexts. + listeners []*ListenerContext +} +``` + +`ListenerContext` wraps a Listener and provides helper methods for setting conditions and other status information on +the associated Gateway. + +```go +type ListenerContext struct { + // The Gateway listener. + *v1beta1.Listener + + // The Gateway this Listener belongs to. + gateway *v1beta1.Gateway + + // An index used for managing this listener in the list of Gateway listeners. + listenerStatusIdx int + + // Only Routes in namespaces selected by the selector may be attached + // to the Gateway this listener belongs to. + namespaceSelector labels.Selector + + // The TLS Secret for this Listener, if applicable. + tlsSecret *v1.Secret +} +``` + +`RouteContext` represents a generic Route object (HTTPRoute, TLSRoute, etc.) that can reference Gateway objects. + +```go +type RouteContext interface { + client.Object + + // GetRouteType returns the Kind of the Route object, HTTPRoute, + // TLSRoute, TCPRoute, UDPRoute etc. + GetRouteType() string + + // GetHostnames returns the hosts targeted by the Route object. + GetHostnames() []string + + // GetParentReferences returns the ParentReference of the Route object. + GetParentReferences() []v1beta1.ParentReference + + // GetRouteParentContext returns RouteParentContext by using the Route + // objects' ParentReference. + GetRouteParentContext(forParentRef v1beta1.ParentReference) *RouteParentContext +} +``` + +[message bus]: watching.md diff --git a/docs/v0.2.0/design/roadmap.md b/docs/v0.2.0/design/roadmap.md new file mode 100644 index 00000000000..d6ec649e4a2 --- /dev/null +++ b/docs/v0.2.0/design/roadmap.md @@ -0,0 +1,60 @@ +# Roadmap + +This document serves as a high-level reference for Envoy Gateway users and contributors to understand the direction of +the project. + +## Contributing to the Roadmap + +- To add a feature to the roadmap, create an [issue][issue] or join a [community meeting][meeting] to discuss your use + case. If your feature is accepted, a maintainer will assign your issue to a [release milestone][milestones] and update + this document accordingly. +- To help with an existing roadmap item, comment on or assign yourself to the associated issue. +- If a roadmap item doesn't have an issue, create one, assign yourself to the issue, and reference this document. A + maintainer will submit a [pull request][PR] to add the feature to the roadmap. __Note:__ The feature should be + discussed in an issue or a community meeting before implementing it. + +If you don't know where to start contributing, help is needed to reduce technical, automation, and documentation debt. +Look for issues with the `help wanted` label to get started. + +## Details + +Roadmap features and timelines may change based on feedback, community contributions, etc. If you depend on a specific +roadmap item, you're encouraged to attend a community meeting to discuss the details, or help us deliver the feature by +contributing to the project. + +`Last Updated: October 2022` + +### [v0.2.0][v0.2.0]: Establish a Solid Foundation + +- Complete the core Envoy Gateway implementation- [Issue #60][60]. +- Establish initial testing, e2e, integration, etc- [Issue #64][64]. +- Establish user and developer project documentation- [Issue #17][17]. +- Achieve Gateway API conformance (e.g. routing, LB, Header transformation, etc.)- [Issue #65][65]. +- Setup a CI/CD pipeline- [Issue #63][63]. + +### [v0.3.0][v0.3.0]: Drive Advanced Features through Extension Mechanisms + +- Global Rate Limiting +- AuthN/AuthZ- [Issue #336][336]. +- Lets Encrypt Integration + +### [v0.4.0][v0.4.0]: Manageability and Scale + +- Tooling for devs/infra admins to aid in managing/maintaining EG +- Support advanced provisioning use cases (e.g. multi-cluster, serverless, etc.) +- Perf testing (EG specifically) +- Support for Chaos engineering? + +[issue]: https://github.com/envoyproxy/gateway/issues +[meeting]: https://docs.google.com/document/d/1leqwsHX8N-XxNEyTflYjRur462ukFxd19Rnk3Uzy55I/edit?usp=sharing +[pr]: https://github.com/envoyproxy/gateway/compare +[milestones]: https://github.com/envoyproxy/gateway/milestones +[v0.2.0]: https://github.com/envoyproxy/gateway/milestone/1 +[v0.3.0]: https://github.com/envoyproxy/gateway/milestone/7 +[v0.4.0]: https://github.com/envoyproxy/gateway/milestone/12 +[60]: https://github.com/envoyproxy/gateway/issues/60 +[64]: https://github.com/envoyproxy/gateway/issues/64 +[17]: https://github.com/envoyproxy/gateway/issues/17 +[65]: https://github.com/envoyproxy/gateway/issues/65 +[63]: https://github.com/envoyproxy/gateway/issues/63 +[336]: https://github.com/envoyproxy/gateway/issues/336 diff --git a/docs/v0.2.0/design/system-design.md b/docs/v0.2.0/design/system-design.md new file mode 100644 index 00000000000..731cb0925b0 --- /dev/null +++ b/docs/v0.2.0/design/system-design.md @@ -0,0 +1,171 @@ +# System Design + +## Goals + +* Define the system components needed to satisfy the requirements of Envoy Gateway. + +## Non-Goals + +* Create a detailed design and interface specification for each system component. + +## Terminology + +* Control Plane- A collection of inter-related software components for providing application gateway and routing + functionality. The control plane is implemented by Envoy Gateway and provides services for managing the data plane. + These services are detailed in the [components](#components) section. +* Data Plane- Provides intelligent application-level traffic routing and is implemented as one or more Envoy proxies. + +## Architecture + +![Architecture](../images/architecture.png) + +## Configuration + +Envoy Gateway is configured statically at startup and the managed data plane is configured dynamically through +Kubernetes resources, primarily [Gateway API][gw_api] objects. + +### Static Configuration + +Static configuration is used to configure Envoy Gateway at startup, i.e. change the GatewayClass controllerName, +configure a Provider, etc. Currently, Envoy Gateway only supports configuration through a configuration file. If the +configuration file is not provided, Envoy Gateway starts-up with default configuration parameters. + +### Dynamic Configuration + +Dynamic configuration is based on the concept of a declaring the desired state of the data plane and using +reconciliation loops to drive the actual state toward the desired state. The desired state of the data plane is +defined as Kubernetes resources that provide the following services: + +* Infrastructure Management- Manage the data plane infrastructure, i.e. deploy, upgrade, etc. This configuration is + expressed through [GatewayClass][gc] and [Gateway][gw] resources. The `EnvoyProxy` [Custom Resource][cr] can be + referenced by `gatewayclass.spec.parametersRef` to modify data plane infrastructure default parameters, + e.g. expose Envoy network endpoints using a NodePort service instead of a LoadBalancer service. +* Traffic Routing- Define how to handle application-level requests to backend services. For example, route all HTTP + requests for "www.example.com" to a backend service running a web server. This configuration is expressed through + [HTTPRoute][hroute] and [TLSRoute][troute] resources that match, filter, and route traffic to a [backend][be]. + Although a backend can be any valid Kubernetes Group/Kind resource, Envoy Gateway only supports a [Service][svc] + reference. + +## Components + +Envoy Gateway is made up of several components that communicate in-process; how this communication happens is described +in the [Watching Components Design][wcd]. + +### Provider + +A Provider is an infrastructure component that Envoy Gateway calls to establish its runtime configuration, resolve +services, persist data, etc. As of v0.2, Kubernetes is the only implemented provider. A file provider is on the roadmap +via [Issue #37][]. Other providers can be added in the future as Envoy Gateway use cases are better understood. A +provider is configured at start up through Envoy Gateway's [static configuration](#static-configuration). + +#### Kubernetes Provider + +* Uses Kubernetes-style controllers to reconcile Kubernetes resources that comprise the + [dynamic configuration](#dynamic-configuration). +* Manages the data plane through Kubernetes API CRUD operations. +* Uses Kubernetes for Service discovery. +* Uses etcd (via Kubernetes API) to persist data. + +#### File Provider + +* Uses a file watcher to watch files in a directory that define the data plane configuration. +* Manages the data plane by calling internal APIs, e.g. `CreateDataPlane()`. +* Uses the host's DNS for Service discovery. +* If needed, the local filesystem is used to persist data. + +### Resource Watcher + +The Resource Watcher watches resources used to establish and maintain Envoy Gateway's dynamic configuration. The +mechanics for watching resources is provider-specific, e.g. informers, caches, etc. are used for the Kubernetes +provider. The Resource Watcher uses the configured provider for input and provides resources to the Resource Translator +as output. + +### Resource Translator + +The Resource Translator translates external resources, e.g. GatewayClass, from the Resource Watcher to the Intermediate +Representation (IR). It is responsible for: + +* Translating infrastructure-specific resources/fields from the Resource Watcher to the Infra IR. +* Translating proxy configuration resources/fields from the Resource Watcher to the xDS IR. + +__Note:__ The Resource Translator is implemented as the `Translator` API type in the `gatewayapi` package. + +### Intermediate Representation (IR) + +The Intermediate Representation defines internal data models that external resources are translated into. This allows +Envoy Gateway to be decoupled from the external resources used for dynamic configuration. The IR consists of an Infra IR +used as input for the Infra Manager and an xDS IR used as input for the xDS Translator. + +* Infra IR- Used as the internal definition of the managed data plane infrastructure. +* xDS IR- Used as the internal definition of the managed data plane xDS configuration. + +### xDS Translator + +The xDS Translator translates the xDS IR into xDS Resources that are consumed by the xDS server. + +### xDS Server + +The xDS Server is a xDS gRPC Server based on [Go Control Plane][go_cp]. Go Control Plane implements the Delta xDS Server +Protocol and is responsible for using xDS to configure the data plane. + +### Infra Manager + +The Infra Manager is a provider-specific component responsible for managing the following infrastructure: + +* Data Plane - Manages all the infrastructure required to run the managed Envoy proxies. For example, CRUD Deployment, + Service, etc. resources to run Envoy in a Kubernetes cluster. +* Auxiliary Control Planes - Optional infrastructure needed to implement application Gateway features that require + external integrations with the managed Envoy proxies. For example, [Global Rate Limiting][grl] requires provisioning + and configuring the [Envoy Rate Limit Service][rls] and the [Rate Limit filter][rlf]. Such features are exposed to + users through the [Custom Route Filters][crf] extension. + +The Infra Manager consumes the Infra IR as input to manage the data plane infrastructure. + +## Design Decisions + +* Envoy Gateway consumes one [GatewayClass][gc] by comparing its configured controller name with + `spec.controllerName` of a GatewayClass. If multiple GatewayClasses exist with the same `spec.controllerName`, Envoy + Gateway follows Gateway API [guidelines][gwapi_conflicts] to resolve the conflict. + `gatewayclass.spec.parametersRef` refers to the `EnvoyProxy` custom resource for configuring the managed proxy + infrastructure. If unspecified, default configuration parameters are used for the managed proxy infrastructure. +* Envoy Gateway manages [Gateways][gw] that reference its GatewayClass. + * A Gateway resource causes Envoy Gateway to provision managed Envoy proxy infrastructure. + * Envoy Gateway groups Listeners by Port and collapses each group of Listeners into a single Listener if the Listeners + in the group are compatible. Envoy Gateway considers Listeners to be compatible if all the following conditions are + met: + * Either each Listener within the group specifies the “HTTP” Protocol or each Listener within the group specifies + either the “HTTPS” or “TLS” Protocol. + * Each Listener within the group specifies a unique "Hostname". + * As a special case, one Listener within a group may omit "Hostname", in which case this Listener matches when no + other Listener matches. + * Envoy Gateway does __not__ merge listeners across multiple Gateways. +* Envoy Gateway follows Gateway API [guidelines][gwapi_conflicts] to resolve any conflicts. + * A Gateway `listener` corresponds to an Envoy proxy [Listener][listener]. +* An [HTTPRoute][hroute] resource corresponds to an Envoy proxy [Route][route]. + * Each [backendRef][be_ref] corresponds to an Envoy proxy [Cluster][cluster]. +* The goal is to make Envoy Gateway components extensible in the future. See the [roadmap][] for additional details. + +The draft for this document is [here][draft_design]. + +[gw_api]: https://gateway-api.sigs.k8s.io +[gc]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#gatewayclass +[gw]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#gateway +[hroute]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#httproute +[troute]: https://gateway-api.sigs.k8s.io/concepts/api-overview/#tlsroute +[go_cp]: https://github.com/envoyproxy/go-control-plane +[grl]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/other_features/global_rate_limiting +[rls]: https://github.com/envoyproxy/ratelimit +[rlf]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ratelimit/v3/rate_limit.proto#envoy-v3-api-msg-extensions-filters-http-ratelimit-v3-ratelimit +[crf]: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#filters-optional +[gwapi_conflicts]: https://gateway-api.sigs.k8s.io/concepts/guidelines/#conflicts +[listener]: https://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/listeners#config-listeners +[route]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-route +[be_ref]: https://gateway-api.sigs.k8s.io/v1alpha2/api-types/httproute/#backendrefs-optional +[cluster]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster +[draft_design]: https://docs.google.com/document/d/1riyTPPYuvNzIhBdrAX8dpfxTmcobWZDSYTTB5NeybuY/edit +[cr]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ +[be]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.BackendObjectReference +[svc]: https://kubernetes.io/docs/concepts/services-networking/service/ +[ wcd ]: ./watching.md +[Issue #37]: https://github.com/envoyproxy/gateway/issues/37 +[roadmap]: roadmap.md diff --git a/docs/v0.2.0/design/watching.md b/docs/v0.2.0/design/watching.md new file mode 100644 index 00000000000..b8477a30e2d --- /dev/null +++ b/docs/v0.2.0/design/watching.md @@ -0,0 +1,117 @@ +# Watching Components Design + +Envoy Gateway is made up of several components that communicate in-process. Some of them (namely Providers) watch +external resources, and "publish" what they see for other components to consume; others watch what another publishes and +act on it (such as the resource translator watches what the providers publish, and then publishes its own results that +are watched by another component). Some of these internally published results are consumed by multiple components. + +To facilitate this communication use the [watchable][] library. The `watchable.Map` type is very similar to the +standard library's `sync.Map` type, but supports a `.Subscribe` (and `.SubscribeSubset`) method that promotes a pub/sub +pattern. + +## Pub + +Many of the things we communicate around are naturally named, either by a bare "name" string or by a "name"/"namespace" +tuple. And because `watchable.Map` is typed, it makes sense to have one map for each type of thing (very similar to if +we were using native Go `map`s). For example, a struct that might be written to by the Kubernetes provider, and read by +the IR translator: + + ```go + type ResourceTable struct { + // gateway classes are cluster-scoped; no namespace + GatewayClasses watchable.Map[string, *gwapiv1b1.GatewayClass] + + // gateways are namespace-scoped, so use a k8s.io/apimachinery/pkg/types.NamespacedName as the map key. + Gateways watchable.Map[types.NamespacedName, *gwapiv1b1.Gateway] + + HTTPRoutes watchable.Map[types.NamespacedName, *gwapiv1b1.HTTPRoute] + } + ``` + +The Kubernetes provider updates the table by calling `table.Thing.Store(name, val)` and `table.Thing.Delete(name)`; +updating a map key with a value that is deep-equal (usually `reflect.DeepEqual`, but you can implement your own `.Equal` +method) the current value is a no-op; it won't trigger an event for subscribers. This is handy so that the publisher +doesn't have as much state to keep track of; it doesn't need to know "did I already publish this thing", it can just +`.Store` its data and `watchable` will do the right thing. + +## Sub + +Meanwhile, the translator and other interested components subscribe to it with `table.Thing.Subscribe` (or +`table.Thing.SubscribeSubset` if they only care about a few "Thing"s). So the translator goroutine might look like: + + ```go + func(ctx context.Context) error { + for snapshot := range k8sTable.HTTPRoutes.Subscribe(ctx) { + fullState := irInput{ + GatewayClasses: k8sTable.GatewayClasses.LoadAll(), + Gateways: k8sTable.Gateways.LoadAll(), + HTTPRoutes: snapshot.State, + } + translate(irInput) + } + } + ``` + +Or, to watch multiple maps in the same loop: + + ```go + func worker(ctx context.Context) error { + classCh := k8sTable.GatewayClasses.Subscribe(ctx) + gwCh := k8sTable.Gateways.Subscribe(ctx) + routeCh := k8sTable.HTTPRoutes.Subscribe(ctx) + for ctx.Err() == nil { + var arg irInput + select { + case snapshot := <-classCh: + arg.GatewayClasses = snapshot.State + case snapshot := <-gwCh: + arg.Gateways = snapshot.State + case snapshot := <-routeCh: + arg.Routes = snapshot.State + } + if arg.GateWayClasses == nil { + arg.GatewayClasses = k8sTable.GateWayClasses.LoadAll() + } + if arg.GateWays == nil { + arg.Gateways = k8sTable.GateWays.LoadAll() + } + if arg.HTTPRoutes == nil { + arg.HTTPRoutes = k8sTable.HTTPRoutes.LoadAll() + } + translate(irInput) + } + } + ``` + +From the updates it gets from `.Subscribe`, it can get a full view of the map being subscribed to via `snapshot.State`; +but it must read the other maps explicitly. Like `sync.Map`, `watchable.Map`s are thread-safe; while `.Subscribe` is a +handy way to know when to run, `.Load` and friends can be used without subscribing. + +There can be any number of subscribers. For that matter, there can be any number of publishers `.Store`ing things, but +it's probably wise to just have one publisher for each map. + +The channel returned from `.Subscribe` **is immediately readable** with a snapshot of the map as it existed when +`.Subscribe` was called; and becomes readable again whenever `.Store` or `.Delete` mutates the map. If multiple +mutations happen between reads (or if mutations happen between `.Subscribe` and the first read), they are coalesced in +to one snapshot to be read; the `snapshot.State` is the most-recent full state, and `snapshot.Updates` is a listing of +each of the mutations that cause this snapshot to be different than the last-read one. This way subscribers don't need +to worry about a backlog accumulating if they can't keep up with the rate of changes from the publisher. + +If the map contains anything before `.Subscribe` is called, that very first read won't include `snapshot.Updates` +entries for those pre-existing items; if you are working with `snapshot.Update` instead of `snapshot.State`, then you +must add special handling for your first read. We have a utility function `./internal/message.HandleSubscription` to +help with this. + +## Other Notes + +The common pattern will likely be that the entrypoint that launches the goroutines for each component instantiates the +map, and passes them to the appropriate publishers and subscribers; same as if they were communicating via a dumb +`chan`. + +A limitation of `watchable.Map` is that in order to ensure safety between goroutines, it does require that value types +be deep-copiable; either by having a `DeepCopy` method, being a `proto.Message`, or by containing no reference types and +so can be deep-copied by naive assignment. Fortunately, we're using `controller-gen` anyway, and `controller-gen` can +generate `DeepCopy` methods for us: just stick a `// +k8s:deepcopy-gen=true` on the types that you want it to generate +methods for. + +[watchable]: https://pkg.go.dev/github.com/telepresenceio/watchable diff --git a/docs/v0.2.0/design_docs.rst b/docs/v0.2.0/design_docs.rst new file mode 100644 index 00000000000..4e95a518d1e --- /dev/null +++ b/docs/v0.2.0/design_docs.rst @@ -0,0 +1,12 @@ +Design Docs +=========== + +Learn about the internal details of Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + design/system-design + design/gatewayapi-translator + design/watching + design/config-api diff --git a/docs/v0.2.0/dev/CODE_OF_CONDUCT.md b/docs/v0.2.0/dev/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..5071043e94a --- /dev/null +++ b/docs/v0.2.0/dev/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +## Community Code of Conduct + +Gateway follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md). diff --git a/docs/v0.2.0/dev/CONTRIBUTING.md b/docs/v0.2.0/dev/CONTRIBUTING.md new file mode 100644 index 00000000000..eb4ad7e5a67 --- /dev/null +++ b/docs/v0.2.0/dev/CONTRIBUTING.md @@ -0,0 +1,183 @@ +# Contributing + +We welcome contributions from the community. Please carefully review the [project goals](GOALS.md) +and following guidelines to streamline your contributions. + +## Communication + +* Before starting work on a major feature, please contact us via GitHub or Slack. We will ensure no + one else is working on it and ask you to open a GitHub issue. +* A "major feature" is defined as any change that is > 100 LOC altered (not including tests), or + changes any user-facing behavior. We will use the GitHub issue to discuss the feature and come to + agreement. This is to prevent your time being wasted, as well as ours. The GitHub review process + for major features is also important so that [affiliations with commit access](CODEOWNERS.md) can + come to agreement on the design. If it's appropriate to write a design document, the document must + be hosted either in the GitHub issue, or linked to from the issue and hosted in a world-readable + location. +* Small patches and bug fixes don't need prior communication. + +## Inclusivity + +The Envoy Gateway community has an explicit goal to be inclusive to all. As such, all PRs must adhere +to the following guidelines for all code, APIs, and documentation: + +* The following words and phrases are not allowed: + * *Whitelist*: use allowlist instead. + * *Blacklist*: use denylist or blocklist instead. + * *Master*: use primary instead. + * *Slave*: use secondary or replica instead. +* Documentation should be written in an inclusive style. The [Google developer + documentation](https://developers.google.com/style/inclusive-documentation) contains an excellent + reference on this topic. +* The above policy is not considered definitive and may be amended in the future as industry best + practices evolve. Additional comments on this topic may be provided by maintainers during code + review. + +## Submitting a PR + +* Fork the repo. +* Hack +* DCO sign-off each commit. This can be done with `git commit -s`. +* Submit your PR. +* Tests will automatically run for you. +* We will **not** merge any PR that is not passing tests. +* PRs are expected to have 100% test coverage for added code. This can be verified with a coverage + build. If your PR cannot have 100% coverage for some reason please clearly explain why when you + open it. +* Any PR that changes user-facing behavior **must** have associated documentation in [docs](docs) as + well as the [changelog](./changelogs). +* All code comments and documentation are expected to have proper English grammar and punctuation. + If you are not a fluent English speaker (or a bad writer ;-)) please let us know and we will try + to find some help but there are no guarantees. +* Your PR title should be descriptive, and generally start with a subsystem name followed by a + colon. Examples: + * "docs: fix grammar error" + * "translator: add new feature" +* Your PR commit message will be used as the commit message when your PR is merged. You should + update this field if your PR diverges during review. +* Your PR description should have details on what the PR does. If it fixes an existing issue it + should end with "Fixes #XXX". +* If your PR is co-authored or based on an earlier PR from another contributor, + please attribute them with `Co-authored-by: name `. See + GitHub's [multiple author + guidance](https://help.github.com/en/github/committing-changes-to-your-project/creating-a-commit-with-multiple-authors) + for further details. +* When all tests are passing and all other conditions described herein are satisfied, a maintainer + will be assigned to review and merge the PR. +* Once you submit a PR, *please do not rebase it*. It's much easier to review if subsequent commits + are new commits and/or merges. We squash and merge so the number of commits you have in the PR + doesn't matter. +* We expect that once a PR is opened, it will be actively worked on until it is merged or closed. + We reserve the right to close PRs that are not making progress. This is generally defined as no + changes for 7 days. Obviously PRs that are closed due to lack of activity can be reopened later. + Closing stale PRs helps us to keep on top of all the work currently in flight. + +## Maintainer PR Review Policy + +* See [CODEOWNERS.md](CODEOWNERS.md) for the current list of maintainers. +* A maintainer representing a different affiliation from the PR owner is required to review and + approve the PR. +* When the project matures, it is expected that a "domain expert" for the code the PR touches should + review the PR. This person does not require commit access, just domain knowledge. +* The above rules may be waived for PRs which only update docs or comments, or trivial changes to + tests and tools (where trivial is decided by the maintainer in question). +* If there is a question on who should review a PR please discuss in Slack. +* Anyone is welcome to review any PR that they want, whether they are a maintainer or not. +* Please make sure that the PR title, commit message, and description are updated if the PR changes + significantly during review. +* Please **clean up the title and body** before merging. By default, GitHub fills the squash merge + title with the original title, and the commit body with every individual commit from the PR. + The maintainer doing the merge should make sure the title follows the guidelines above and should + overwrite the body with the original commit message from the PR (cleaning it up if necessary) + while preserving the PR author's final DCO sign-off. + +## Decision making + +This is a new and complex project, and we need to make a lot of decisions very quickly. +To this end, we've settled on this process for making (possibly contentious) decisions: + +* For decisions that need a record, we create an issue. +* In that issue, we discuss opinions, then a maintainer can call for a vote in a comment. +* Maintainers can cast binding votes on that comment by reacting or replying in another comment. +* Non-maintainer community members are welcome to cast non-binding votes by either of these methods. +* Voting will be resolved by simple majority. +* In the event of deadlocks, the question will be put to steering instead. + +## DCO: Sign your work + +The sign-off is a simple line at the end of the explanation for the +patch, which certifies that you wrote it or otherwise have the right to +pass it on as an open-source patch. The rules are pretty simple: if you +can certify the below (from +[developercertificate.org](https://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +then you just add a line to every git commit message: + + Signed-off-by: Joe Smith + +using your real name (sorry, no pseudonyms or anonymous contributions.) + +You can add the sign-off when creating the git commit via `git commit -s`. + +If you want this to be automatic you can set up some aliases: + +```bash +git config --add alias.amend "commit -s --amend" +git config --add alias.c "commit -s" +``` + +## Fixing DCO + +If your PR fails the DCO check, it's necessary to fix the entire commit history in the PR. Best +practice is to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) +the commit history to a single commit, append the DCO sign-off as described above, and [force +push](https://git-scm.com/docs/git-push#git-push---force). For example, if you have 2 commits in +your history: + +```bash +git rebase -i HEAD^^ +(interactive squash + DCO append) +git push origin -f +``` + +Note, that in general rewriting history in this way is a hindrance to the review process and this +should only be done to correct a DCO mistake. diff --git a/docs/v0.2.0/dev/DOCS.md b/docs/v0.2.0/dev/DOCS.md new file mode 100644 index 00000000000..fb49b9d55dd --- /dev/null +++ b/docs/v0.2.0/dev/DOCS.md @@ -0,0 +1,63 @@ +# Working on the Envoy Gateway Docs + +The documentation for the Envoy Gateway lives in the `docs/` directory. Any +individual document can be written using either [reStructuredText] or [Markdown], +you can choose the format that you're most comfortable with when working on the +documentation. + +## Documentation Structure + +We supported the versioned Docs now, the directory name under docs represents +the version of docs. The root of the latest site is in `docs/latest/index.rst`. +This is probably where to start if you're trying to understand how things fit together. + +Note that the new contents should be added to `docs/latest` and will be cut off at +the next release. The contents under `docs/v0.2.0` are auto-generated, +and usually do not need to make changes to them, unless if you find the current release pages have +some incorrect contents. If so, you should send a PR to update contents both of `docs/latest` +and `docs/v0.2.0`. + +It's important to note that a given document _must_ have a reference in some +`.. toctree::` section for the document to be reachable. Not everything needs +to be in `docs/index.rst`'s `toctree` though. + +You can access the website which represents the current release in default, +and you can access the website which contains the latest version changes in +[Here][latest-website] or at the footer of the pages. + +## Documentation Workflow + +To work with the docs, just edit reStructuredText or Markdown files in `docs`, +then run + +```bash +make docs +``` + +This will create `docs/html` with the built HTML pages. You can view the docs +either simply by pointing a web browser at the `file://` path to your +`docs/html`, or by firing up a static webserver from that directory, e.g. + +``` shell +make docs-serve +``` + +If you want to generate a new release version of the docs, like `v0.3.0`, then run + +```bash +make docs-release TAG=v0.3.0 +``` + +This will update the VERSION file at the project root, which records current release version, +and it will be used in the pages version context and binary version output. Also, this will generate +new dir `docs/v0.3.0`, which contains docs at v0.3.0 and updates artifact links to `v0.3.0` +in all files under `docs/v0.3.0/user`, like `quickstart.md`, `http-routing.md` and etc. + +## Publishing Docs + +Whenever docs are pushed to `main`, CI will publish the built docs to GitHub +Pages. For more details, see `.github/workflows/docs.yaml`. + +[reStructuredText]: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html +[Markdown]: https://daringfireball.net/projects/markdown/syntax +[latest-website]: https://gateway.envoyproxy.io/latest diff --git a/docs/v0.2.0/dev/README.md b/docs/v0.2.0/dev/README.md new file mode 100644 index 00000000000..e64c2718ced --- /dev/null +++ b/docs/v0.2.0/dev/README.md @@ -0,0 +1,139 @@ +# Developer Guide + +Envoy Gateway is built using a [make][]-based build system. Our CI is based on [Github Actions][] using [workflows][]. + +## Prerequisites + +### go + +* Version: 1.18.2 +* Installation Guide: https://go.dev/doc/install + +### make + +* Recommended Version: 4.0 or later +* Installation Guide: https://www.gnu.org/software/make + +### docker + +* Optional when you want to build a Docker image or run `make` inside Docker. +* Recommended Version: 20.10.16 +* Installation Guide: https://docs.docker.com/engine/install + +### python3 + +* Need a `python3` program +* Must have a functioning `venv` module; this is part of the standard + library, but some distributions (such as Debian and Ubuntu) replace + it with a stub and require you to install a `python3-venv` package + separately. + +## Quickstart + +* Run `make help` to see all the available targets to build, test and run Envoy Gateway. + +### Building + +* Run `make build` to build the Envoy Gateway binary. __Note:__ The binary gets generated in the `bin/` directory + +### Testing + +* Run `make test` to run the golang tests. + +### Running Linters + +* Run `make lint` to make sure your code passes all the linter checks. + +### Building and Pushing the Image + +* Run `IMAGE=docker.io/you/gateway-dev make image` to build the docker image. +* Run `IMAGE=docker.io/you/gateway-dev make push-multiarch` to build and push the multi-arch docker image. + +__Note:__ Replace `IMAGE` with your registry's image name. + +### Deploying Envoy Gateway for Test/Dev + +* Run `make create-cluster` to create a [Kind][] cluster. + +#### Option 1: Use the Latest [gateway-dev][] Image + +* Run `TAG=latest make kube-deploy` to deploy Envoy Gateway in the Kind cluster using the latest image. Replace `latest` + to use a different image tag. + +#### Option 2: Use a Custom Image + +* Run `make kube-install-image` to build an image from the tip of your current branch and load it in the Kind cluster. +* Run `make kube-deploy` to install Envoy Gateway into the Kind cluster using your custom image. + +### Deploying Envoy Gateway in Kubernetes + +* Run `TAG=latest make kube-deploy` to deploy Envoy Gateway using the latest image into a Kubernetes cluster (linked to + the current kube context). Preface the command with `IMAGE` or replace `TAG` to use a different Envoy Gateway image or + tag. +* Run `make kube-undeploy` to uninstall Envoy Gateway from the cluster. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24.0. + +### Demo Setup + +* Run `make kube-demo` to deploy a demo backend service, gatewayclass, gateway and httproute resource +(similar to steps outlined in the [Quickstart][] docs) and test the configuration. +* Run `make kube-demo-undeploy` to delete the resources created by the `make kube-demo` command. + +### Run Gateway API Conformance Tests + +The commands below deploy Envoy Gateway to a Kubernetes cluster and run the Gateway API conformance tests. Refer to the +Gateway API [conformance homepage][] to learn more about the tests. If Envoy Gateway is already installed, run +`TAG=latest make run-conformance` to run the conformance tests. + +#### On a Linux Host + +* Run `TAG=latest make conformance` to create a Kind cluster, install Envoy Gateway using the latest [gateway-dev][] + image, and run Gateway API conformance tests. + +#### On a Mac Host + +Since Mac doesn't support [directly exposing][] the Docker network to the Mac host, use one of the following +workarounds to run conformance tests: + +* Deploy your own Kubernetes cluster or use Docker Desktop with [Kubernetes support][] and then run + `TAG=latest make kube-deploy run-conformance`. This will install Envoy Gateway using the latest [gateway-dev][] image + to the Kubernetes cluster using the current kubectl context and run the conformance tests. Use `make kube-undeploy` to + uninstall Envoy Gateway. +* Install and run [Docker Mac Net Connect][mac_connect] and then run `TAG=latest make conformance`. + +__Note:__ Preface commands with `IMAGE` or replace `TAG` to use a different Envoy Gateway image or tag. If `TAG` +is unspecified, the short SHA of your current branch is used. + +### Debugging the Envoy Config + +An easy way to view the envoy config that Envoy Gateway is using is to port-forward to the admin interface port +(currently `19000`) on the Envoy deployment that corresponds to a Gateway so that it can be accessed locally. + +Get the name of the Envoy deployment. The following example is for Gateway `eg` in the `default` namespace: + +```shell +export ENVOY_DEPLOYMENT=$(kubectl get deploy -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward the admin interface port: + +```shell +kubectl port-forward deploy/envoy-${ENVOY_DEPLOYMENT} -n envoy-gateway-system 19000:19000 +``` + +Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. + +There are many other endpoints on the [Envoy admin interface][] that may be helpful when debugging. + +[Quickstart]: https://github.com/envoyproxy/gateway/blob/main/docs/user/quickstart.md +[make]: https://www.gnu.org/software/make/ +[Github Actions]: https://docs.github.com/en/actions +[workflows]: .github/workflows +[Kind]: https://kind.sigs.k8s.io/ +[conformance homepage]: https://gateway-api.sigs.k8s.io/concepts/conformance/ +[directly exposing]: https://kind.sigs.k8s.io/docs/user/loadbalancer/ +[Kubernetes support]: https://docs.docker.com/desktop/kubernetes/ +[gateway-dev]: https://hub.docker.com/r/envoyproxy/gateway-dev/tags +[mac_connect]: https://github.com/chipmk/docker-mac-net-connect +[Envoy admin interface]: https://www.envoyproxy.io/docs/envoy/latest/operations/admin#operations-admin-interface diff --git a/docs/v0.2.0/dev/releasing.md b/docs/v0.2.0/dev/releasing.md new file mode 100644 index 00000000000..a6d965be92f --- /dev/null +++ b/docs/v0.2.0/dev/releasing.md @@ -0,0 +1,144 @@ +# Release Process + +This document guides maintainers through the process of creating an Envoy Gateway release. + +## Creating a Minor Release + +### Prerequisites + +- Permissions to push to the Envoy Gateway repository. + +### Set Environment Variables + +Set environment variables for use in subsequent steps: + +```shell +export MAJOR_VERSION=0 +export MINOR_VERSION=3 +export GITHUB_REMOTE=origin +``` + +1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. +2. Create a topic branch to create the release notes and release docs. Reference previous [release notes][] for additional details. +3. Sign, commit, and push your changes to your fork and submit a [Pull Request][] to merge the changes listed below + into the `main` branch. Do not proceed until all your PRs have merged and the [Build and Test][build-and-test GitHub action] has completed for your final PR: + + 1. Add Release Announcement. + 2. Add Release Versioned Documents. + + ``` shell + make docs-release TAG=v${MAJOR_VERSION}.${MINOR_VERSION}.0 + ``` + +4. Create a new release branch from `main`. The release branch should be named + `release/v${MAJOR_VERSION}.${MINOR_VERSION}`, e.g. `release/v0.3`. + + ```shell + git checkout -b release/v${MAJOR_VERSION}.${MINOR_VERSION} + ``` + +5. Push the branch to the Envoy Gateway repo. + + ```shell + git push ${GITHUB_REMOTE} release/v${MAJOR_VERSION}.${MINOR_VERSION} + ``` + +6. Tag the head of your release branch with the release tag. For example: + + ```shell + git tag -a v${MAJOR_VERSION}.${MINOR_VERSION}.0 -m 'Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0 Release' + ``` + + __Note:__ The tag version differs from the release branch by including the `.0` patch version. + +7. Push the tag to the Envoy Gateway repository. + + ```shell + git push origin v${MAJOR_VERSION}.${MINOR_VERSION}.0 + ``` + +8. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +9. Confirm that the [release workflow][] completed successfully. +10. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +11. Confirm that the [release][] was created. +12. Confirm that the steps in the [Quickstart Guide][] work as expected. +13. [Generate][] the GitHub changelog and include the following text at the beginning of the release page: + + ```console + # Release Announcement + + Check out the [v${MAJOR_VERSION}.${MINOR_VERSION} release announcement] + (https://gateway.envoyproxy.io/releases/v${MAJOR_VERSION}.${MINOR_VERSION}.html) to learn more about the release. + ``` + +If you find any bugs in this process, please create an issue. + +## Creating a Release Candidate + +### Prerequisites + +- Permissions to push to the Envoy Gateway repository. + +### Set Environment Variables + +```shell +export MAJOR_VERSION=0 +export MINOR_VERSION=3 +export RELEASE_CANDIDATE_NUMBER=1 +export GITHUB_REMOTE=origin +``` + +1. Clone the repo, checkout the `main` branch, ensure it’s up-to-date, and your local branch is clean. +2. Tag the head of the main branch with the release candidate number. + + ```shell + git tag -a v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} -m 'Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} Release Candidate' + ``` + +3. Push the tag to the Envoy Gateway repository. + + ```shell + git push v${MAJOR_VERSION}.${MINOR_VERSION}.0-rc.${RELEASE_CANDIDATE_NUMBER} + ``` + +4. This will trigger the [release GitHub action][] that generates the release, release artifacts, etc. +5. Confirm that the [release workflow][] completed successfully. +6. Confirm that the Envoy Gateway [image][] with the correct release tag was published to Docker Hub. +7. Confirm that the [release][] was created. +8. Note that the [Quickstart Guide][] references are __not__ updated for release candidates. However, test + the quickstart steps using the release candidate by manually updating the links. +9. [Generate][] the GitHub changelog. +10. Ensure you check the "This is a pre-release" checkbox when editing the GitHub release. +11. If you find any bugs in this process, please create an issue. + +## Announcing the Release + +It's important that the world knows about the release. Use the following steps to announce the release. + +1. Set the release information in the Envoy Gateway Slack channel. For example: + + ```shell + Envoy Gateway v${MAJOR_VERSION}.${MINOR_VERSION} has been released: https://github.com/envoyproxy/gateway/releases/tag/v${MAJOR_VERSION}.${MINOR_VERSION}.0 + ``` + +2. Send a message to the Envoy Gateway Slack channel. For example: + + ```shell + On behalf of the entire Envoy Gateway community, I am pleased to announce the release of Envoy Gateway + v${MAJOR_VERSION}.${MINOR_VERSION}. A big thank you to all the contributors that made this release possible. + Refer to the official v${MAJOR_VERSION}.${MINOR_VERSION} announcement for release details and the project docs + to start using Envoy Gateway. + ... + ``` + + Link to the GitHub release and release announcement page that highlights the release. + +[release notes]: https://github.com/envoyproxy/gateway/tree/main/release-notes +[Pull Request]: https://github.com/envoyproxy/gateway/pulls +[Quickstart Guide]: https://github.com/envoyproxy/gateway/blob/main/docs/user/quickstart.md +[build-and-test GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/build_and_test.yaml +[release GitHub action]: https://github.com/envoyproxy/gateway/blob/main/.github/workflows/release.yaml +[release workflow]: https://github.com/envoyproxy/gateway/actions/workflows/release.yaml +[image]: https://hub.docker.com/r/envoyproxy/gateway/tags +[release]: https://github.com/envoyproxy/gateway/releases +[Generate]: https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes diff --git a/docs/v0.2.0/dev_docs.rst b/docs/v0.2.0/dev_docs.rst new file mode 100644 index 00000000000..88a0b367f01 --- /dev/null +++ b/docs/v0.2.0/dev_docs.rst @@ -0,0 +1,13 @@ +Developer Docs +============== + +Learn how to contribute to Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + dev/CODE_OF_CONDUCT + dev/CONTRIBUTING + dev/README + dev/DOCS + dev/releasing diff --git a/docs/v0.2.0/get_involved.rst b/docs/v0.2.0/get_involved.rst new file mode 100644 index 00000000000..cd4a70b07ce --- /dev/null +++ b/docs/v0.2.0/get_involved.rst @@ -0,0 +1,9 @@ +Getting involved +================ + +We welcome contributions from the community. Please carefully review the +`project goals `_ +and the +`code of conduct `_ +before diving in. + diff --git a/docs/v0.2.0/images/architecture.png b/docs/v0.2.0/images/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..1d4131fbea77efd32a37cd054d7b8639aee52016 GIT binary patch literal 449265 zcmeFa2RN2}`#6p$qoha)k&#h$vbiOSvLa-cos~Tj?zD|$WMpP%R>-E3l_Z4hvNN*x z_??%^=%u&k`+on&@jd>>`y2<|kL$iZ*ZDd3`kdE2IhoVD@DJi+U|{T$Jag(i2FA`F z3=I4(-0koenlgVr3=B+^k(ijAq?j1BoTa(Gk%=A#2IE5=ZSB2hjxm;9x}>dL_Wmdn zzNOvyyLUa#YrB*bl~9)yy(%iDei*2(-f@7aW1He@j41h6FO7G4P|rPQElc{)Z8My_ zqwNp}XFz7MfJW{v1tI(nDM=QY?KLj0yiC~<@i)if=a&U{OStNQlNh+J_w? z#m^|2%!v81Y8J;|JgA?H9^b2oNb15!<=OgLd3H{A&db^uw|GuX8gN|Y<-8wy@~(dt z7rTlJHfO=%;wJ-S2kNzSwTWuP?j{gmxD|P!zhD@MUY4h6!SrZXxnJwyNo^8FvweF# z)$+2-BRy^H_P0eNFZ-64+pU(Dmvdngvq$924`bl#Y97NL>tXv4$Cj1CvlCOT2z+}YOw(@r@42(NQ7}!6*BLn{;|9Qg?a?g){xB1-0*a82y z7k=y`uzr1eXHUeoU!U>2;2MUgf|#Tv{Hvg2si$XVWoT~AlW~6x{(yJ&%tb2<3{raJ z2UGGq-3YATXr!oOts;GnU&q{(^^&gnWj$7V)2qlj7=rfv@X=Jy`VzIhsfn2tzr7I6 z$~XAoGxBFP8tRp=SQ`t`s7T9Ei{#u%Sj{aB*f{w3_}JJv**G~_;2SJf4rbPu>{-mLXn!p7Yn@YiRyvkO zSFMfA&8U%eFI_gbu@<7CK_0aF-w!zT?2Xo*WM=hqTd+YkhO2 zBYQoQi>HiC0cY?GVNM=iUcr?c{^iozBiG%kvUV#M7dz+rTi0Fs{niUsdX{45rtnZ} z;WdK&ym|e_pEnA!A$wnkiyw$y`72;r7+;WWm1@HHXTJ6N0wF1kPRT05zrbe5e>j%# zi}A<5@OhiLE-Q795C#ScL-LfUqCMtdE1vg-PoW|sRMgb>B;@umx*5}9eYDgk;`ti? z@l?tEPHO&hLCmz&H@FV&!OilxDeZliDd6(G%OM{3?h(j%yvw4K`g(v^p7x`XsQjrN zG`Qcg(74MjRfXo$L;lk_TGIp5!m0NidqeC#WcNB%Jq-Rh|9&WKku7}WFv=AJ6AK%c zi29#>pk_c5-r(bN$PWA;-R_Etan<&6`)AKwg_L?X=Bp6FYU}?Y@YI2wJOAl^e?GfR z48_i{`?$tG#@CPgaeH{T{c~Ke+&e}@R9U`FdjJ1`R>=J~_^SSAgu*}zVq;s4p=xja z&+Z2@bN!dhKiTKs3)z3k{7*&ezta3qRKkCy`5#-m|0?Z&qWS-8oBvwZ|I&2-wax#; zLHMr=_V2xjU!3*B*Zi*whTtQD64#`?K&U=hZGWP2s#k__gT#YFLQl-w3r63vIdkOL z4mA-fY2W`V_gqRA#U3{?Rd;^_q9aaWhdJ~F&3t^|OfubSrfBfYm1DUB;JCyS96^vLZq!%vu7C8!9JTb8OsO?b?ML zr-vy^4_z|}v8-Tv)}VP!FQq(Vte9A~c(ym3syWj%anNz()sE5+&QmY?8lUN;h;chE zE>5>oDa4+WOeGVvzuID5D{YjY@Y+{am`tbCOS<=S-DHMKyHQ-|<(I9t?!&FQ%G=X! z?O6B78V)07*>CnGW_0%p9eJu=4&~21joINUe_7;_odDt|aM&E{%wp)D=I=YR! zeR*kGr$NK+eYVSmJ6en1lr6sAV^F%FHYkAiVJ9X3as8?gvOAQvDw>@#VS;=;RiTgj z=i9D*f4je6`<7{$dLeLlDz6RkelUZHNO=64VZ7rT1>n?hU)fIW(&{)ofmP21}SB zO{b z5RfI`9a7rJZ4rmj?Tf>97A0f`#cJmHpQ`CUW?S_(+O=Olx@^*zc39J8aq>(XNo_R) zULl>iX%L5wgjm4wM$JBLnNOaw!n0j7z{z_wa;y`sTp3eij|+~!Ha+~z@Z%HS@k*{l za{mq7x&@f3(!koE;#==-L}MMB_f3oKyAM$;&lfJM_1*wJ6JgEn@~LQ_pZ>1JP~>yd z>JgjPA)c12VuC;mJEJZ<(m9*2Z-+S(6ZNn#7+viVlWJCt5~@Z0&gMc~ML_wV1Y zm=#)H7=LEnU(=XlV;nkAB|JZv)%{5ATRKl&b(kC9Zov`M1h8pjj9wFQAy7S`hJAMF z4b>8n%hGr_R|B2DS!>Q)kQK`InG&9F3$f^pP;a2_uS*eWfc#sN-259oK|@9DcaK;& zbj!T$za_+RHqc*kcUf$3Np@;}UnTH8WHvdng3_*K5Al(6ce#!0by;T@dPS&q3r)yT zT%Y(DG*!YpiDPA!JCu2Vfqoq_26Ad;_$v$EwKPKUS`j7+Nc0!6kqxOdtr>U-!yt%#0eFv2jyVY#L z{4jZH!Q2(=Z*@$yZE`V0uR^*sAM==}?Cf3WVh|~B;SXxB@IM-5)ms%bHOJh2urlGf z{ou3GuUbppT9-jxU6A%~HwdwiXmOq&3OcdxF9H(uYkb@ulKO_e-y}d34SY;DH#I~~ ze!`?o-Ksa#uB1OMJk01_%yZ|(Io^qCu>ouzONUPB$Qyj+{602RmSu-WD~_h>KFYZI z%_$b-FA3KjvSXPEY2$W$rxw*y4NsbgYFVR2l}>52kLm;V+gaS34J}B`~6UPv$GFR;qj?Gm9vRe#;N(FJ1DNAZhwo9@~Q~b zcUJtHe4C*KSO-!DZtCAKr`qx2gRnc8I=)P^)?iJih0)T|RF}np^tP7Qc@E>3zZTyM zx@49=S&!I9goLWJfn55Pfmx#^EM0oIOO9Ot^`nzbXArtVE^c``dO4vRi#DTx>3UJK3NWZe%PIQ*iaY z2N~Dbv3G3dmW^|BZMMy(9G8pl9+@BR!vTw?;nXY8Uar1$M`YnJp~%9>Iklnt+AOcj19lPH+!3O(~^URl-xrV|pBiXh? zT#gfcU>TVopD?|TxCvaRIGu-H*6Fg&Ec@33b4|M}+aZ?KY96FJ&FgH>jY36itEb`; zo(K2wv^!bZywQ>}FI==m9Fp4Dlbhpe13u*r08L+K9rWMGcE_2$Z|anIkoji!lrw~T z(0cLG)Yi9DfDR`%u1}J@LniEG-(pcpX_Oy%(iq!{^q2}!Z@7!IK0aIFg5|_O16!?9 zE=`~U3nvIbpzw7^1Tf=<6wkK&YX(hU-kO*&ql8 zhU02S5#me8iY!_mQ_DP-WA|Ni)}T5<^clEU$N5YEI3GEgy;TWw)&un{!ZRIih(C2f zA;0|w-=|~k^KH|Yo6?Ow?Q&tl>(S-?8_00>0jn{lzWR;T7O|4|RVFMCOTWyNVDHW9 z^m30m8xZTu*B9Y-8SDfg>IQI-2mIXHyHS2NZ1Cr$2o}VPIoT?-;VBV=IH3K;=i8Ts z3+6thmc)fS2Z!C-pnnwK;r1x1G49!H(T(iTbP@~;OI@q)2Fh1xMufQWNgn7mU!F4W z1aI_1h})e*j*K#(?#`)QMFh5)68i`$caE9cq<5~4m6_!+YsfRAhO3TJ=u z!t6MgK?i~^uv^D#>*d7R<5VTssFo3!O+%Pb#F$)bsseykLiAG!U1>A^B*%~gA=LTN1y@R+H~h!ONC zTH*btgl}=<%Xgf-KRe!Ans(`pr7!`Ah%Wq)N^QXJMn~l5C3GefF^mXorZuMZiI`lmYS-I*E&dfb~I z&SkS2VsjpK=bfxm;4+;>bmS-aVe#9PjW%w`^e}Z`E$O#m$qnM7$5wsFjSoUAH4pFt z;X#O-yN!^Sx;{nYvF38J`Sqhq<08vk$-Qbsl?jiCD-$y1Xf`gq84}ZPfz?-w&t=%4 z68!Q}2|KKOc7=iFJQvE(-8zt>ky~<>EuUn5=%rTEBUw=p66ya~NQf#OmKUdwwg-=c z7aTn=WZ6}Q#CfV8;z0E#9|=@2jXbph37hSJ$dCXmNT!^f>;{d%cnxid+e7zQ(5NOl zYwlZO?cx&RR3qkLFn_wuNG$~iB!S~U6+yqV>;@23ex}aChGBfoFEc^X3<&FczwE6CqTpY^QGxu!LEKv;ZSBXI1YP)GAR>0CD~B{R0x~P?7L?U<2@4v`!%fP7dm!`oEgX_@tI{n z`vr%C$G|lO0eP_Wd)8%OUMs}M@8`WX`0O+ik|y(9$eO4*t1~tDIq02_w<4reI8EOt z;PxCpEvvPG*qa&~)Hn{dRj|1s_QtPzzbFEKjA(ge_p*1t-9Ylol8XkR{@{~!cRmpD zVsrXJ@XThQ-Z~IGLIJbv?nmiua*?o^av>j^#fdl(Y4d=a3Q8&JWRi{a4YHieR#kiq zwL`E8PX!r1dc;S$bcd4kgV3}(|GDL*#i%DwXkZ_Pp4wG7`_+k%7*?Nopg!3haW4Nu zM?}-HKjzGhL);?1N2KpVFkf$SMQPTqa9(32YUD`C`g&iPU_A_7Z z)H*8M0tYa}x<+!ejP^maLO-OPx*y;3c}Y9_S5c%U82gcu;omgUH(W4KeBf=9Hor8N zbs^`sHf(WOUNp+jzJ)_TiX@}L$$EMoo2QIt236uL7uM96S4!R%0H_ z2OjoQr$pJNtZla-crFGP(KmanlIw#<`0_$;P-c&!4^QV#3SM@u;+KVsLr4bA&&W8b z=HRQ4#i?dCrBuyO86BXP$K*gmoDI%KX88@=AKH8C_!6SXw-C(Mw$DS539_e(M7^fQ zGG9Whs_vK z=7%V@6Ov5ECaTV~n}WYV(-RqrA~99*agvRHE)%E5i$+v`Ej^x9Wia>A+DEc_pP%a? zIgB%Hl?s#n?S(FM<~Ke+l!aZCUzg?ma?^g|VCBRNTnL}qH3##9} z-LFL<84;Tu4tX0|LV>}w#&jbEe#O16l5KS>$v0Eur*iESJZ3Eigd8UfpAHZ%3jd1F z8oa1Pgj2G6g3mq1-w5O8QvU@US9}9w-Hn8-jxU0Q8)f-pP2st|sEHiJo`Jj{a@)QO z6eNqu=jSn;Dq~;W+4OUcpgrlbp3#P%#IZW$(DjhFKiZd5A&}2XU)}bzUQlK%vw8lc z62;Xb%!f%C-)cebR?-!HQ4dnCR1D0|bUsL>h|}+d3`_&wtGb2!0wiDu>Lu^wgZvAN z)j<0~56|eQRA=tL43Kz{0O|9=ppEY6aVB=BnUSo4lw8h+T?g2yotNj^dlyK9o{F-9 z6PUF$H<0+$zr9>d6&Rc{cR0s5Ha}7d$!jbknKp#&90!m{_vwYi%a90>X||2NI@cdp z>O+5)Ghk|sgZVD+J)U@Ai; z+fq-Tu$Xp^FrZYU2ybgaxZv0b-z_Rw7lrc+cHA7A|LB636~MCC<7`Pl7U zT+gK^&+Hf|KQ!77k@57tjBI>=*$++~3y92Z|E`C$$gqa09Z2F1NllEx zhlXrkBiWNj?@(Btf&58i)pyA^bCUx)ZFvs<=H;`%P~X(8+8lb{zP?2;kj_&tSEemv z*^b_}Lqc3!oc|Wx-3drdA)ahaCM6Rfh|2=Wd(3|?)ao7U+QliEWdu>^#{`8q{z8Ub z_gZ(m%)F86oNl|8TE!g~YOC4;-h@q)U36rDc2!e`ag^QnHh~%;un?q2m6GmuzY8cO zV9O(d@0-&NB!9Ph&)rZwGuk;YFG;>}Q=3a6CfHv5yT*nar7$fDi-7dlU09eMGBv8^ zh%-ZvWWsqNO8p9U*+XP`_Je<_R8<}+&=z5t9L9fi}`v( ze6tsA8Z`FrSHCJ6YkHblWd8F9=60{~S39Wct~oTMXy|}jnVJqvtd$3@<-dZxK7A0$HW}!rq!dY!oo#f!a9P4}5;UEpgS5|1@RSnyU%;3X1YZzH z{^hYCi#BkY>?c1dcY5-)Ati)jkZ2@SkZeWL-Jla(5DlYSZ*q9k5b8G>PG*1z)Ukf} zFM_Y%?nG9XI|<5cibPTM0{wWY=R-n#b1 zgFRPvhq2|2oOGJ{Y_L-5vb&&m;oAE-a43$Z$~xzQDdWO$-_KT6cVVM)S<~U@7H}Rd z$1S-xYAQxDG@?3$%NnY;UE`)|JA0@j@#(b@u4~wWjlq-g7vJ^1s;58usE&dr&eF2b zWiHO8BH_V2cjc!;g5PrnQnpi_NpWP%{+8R*ZX>BSW-!)OzH3|E_}L+zopn}{ZqoPS z=Jgy@o5#VNR|*zu+7Mm|^PH{55JmBUqVf`&LCFF$z0DxSW#JNB_z)0acT4ZQn4 zBt01SnT*#sdS~1x1RWf%zT}+vS@t(SaV3uT^KCzh=pNZxpH@nV;XC&R)%r+h<4}>yF|NI zk=>t2C;b=3y)Rg3K0lOUTy<8d*rD99E`^Wn;+|gm0r}jXJYDgSfTYty!Q(#rjP8y5 zc*u{_1}Z3sZ3ST$E08@m=n%-Mp#>Vso7~2g>Pv5o&Mw?h+wH7mzW>7&_c^IUA!Sc_ z*)Be4;_jtC`ZZ5i=t=`Tkl+l$(C&4X}y)fS$yy&+T z*P#octjLQ8&@p0xOMaW@l^f1Q^4+BW8xSCSscGt^&(7ru)Sr&fXWla0Vox`N{(xNpk7yx3Np6wF&FHfjR^Go;)abTsN60i;b# zzP<8Jk~S7O@YVU^GT@JEOZ>4kjQdQa&J~K}TKD@n+)e!!nIF7+oZggs0PNnBwr;ZY z2(>AT_rXtHPCawfLl87v@$kN8(U^9L&&IN_B>R5rIMn=PmJcO5c4U3~o*e9AN8kO_ z@sr)y^hL;uknf0}GzKp=`4MI6k!8Uwt@^|aAuUK}OhisRwK_kkVZ(aGS=m3=b|^SB z>mUeYQ#R_;vWQ&9&Q~GKc>fml>$LjPiE{A-y?q6S4ofj3KHlE2#zq7-LJS<;pqlQ3bpoUzi zLf;i?r5qC9vMvz67c#FtBhni*)kat&ioq|Ia&lvNu%>j@>l#%JaM(VTw|Z}Wc?Bv| zGg8*@%+1|U)>yp()emgyH{7O5wtnG_`@ukafY)(bY^zx2xEg@*`R4Xbtoy3Lt`QDH%2*&>X*V?UALLHP<6U3hUsr3S7gkKrN#eg``tY)uo z;&Esfxu&PAg8BQ0e{Wl`anscRblY5;BQl$VF3t=-Ml`vO>PF&j{IaSl;4ntq^Yq3) z{~|;;C_9B=hN3w>9_mT~~B#(hFc{^-9-aBUX3vP7CDtvTGG6_AiX`gM#n{v?;H3TeZK`$*tCGhMb5kBAY; zJnn5#Jo^`As`cBC7zD*rVfKzC5CeUg=zxH&i4!EQ`W2BL0O(JX9Qdn-jSZ-fTL3lm zw(zw68CMh@qCPufRW~n^H=&Eb*dYb={&SE-t3%p;RF$`+i6!uT`TKXD;Aw_?&``s9 z3$bO*vz`y(R1hOdZwYDy5tda?q-dPp`9y2~tf2e3de3j}T^ zC%3If*P8l49fDmtMD%avcOL|0YlK}Y3&1k^0p%lgJ%5Wze~F_f1nI|PJ9#!{cUD!c zcP`vLtnauv7(e*IFaUrVvTQyVyn#P{Jpuc^3NXrDp3=J2zijd$JsKW#_R2x`&DX#^ z1r8g|21>8O_~S#xDR|1c6HT0(JADzKfYnqzn>;sp_Uh)+lL3h4xm&KP21S zZTLx*J@6pXOUH3HUxVlj=>PNhe=L&301qlO%^TZnzL)AbpnJ0Uu-E37e*oKrkVU$& z)o*jUN+G;)lc%BPPh_bh0oNfRa;uivVCaXcbrPdpCF3R^4EG~Edin7=pUvTDdI%^L zXga3wr$kRR1!A`P`nM1z>~ElN>bO6GP@-=Ts6%`6DCFj1rn8}D4NaVfwpOzL6Ney> z{nE)iXiw-@0sZX>=_=a)6Q+fjLB<0z?J)}3-<~F3hGN{gEj8nsI{Td_f3L*1W*a@o zd@0zfnZ3fsC2=c?mh`<0&9wcBP%b4Fx;FKH0xDoRj=>vzo3QIv24Y3hA{>C1-;dWm z2DCZgIZfxbd)l=*FboM?n;L9{cFdX_tSw}I>t)xe6uBE%z6`QnyU0&OA0@7oWFpxb z$D0d&wHczm-YW$$Vq(Q0IXtOcBbLp-`kM}4dy*eyIApSi^K6Kzt7EY;-1w9=9rPw3 zWiH*@)$gnTH#)hT}|q1&9&Rw>fa)YkB`3#RVD_? z9IZlU4(Mh0yMUyc257|c+K^~-yILl%UG76~(SCh8CSQiHo8(o9qS#g**lI3?=#os+ zuUP=#gk-{=Uf}K80)-9`S$3gg3AW$_Lqsj_p%4K}79ocfpl&Id2PO3W5iWx&52z(> z9bCxaf(|?xe{f@~-zwoMF2cGYcI{1@Tj3RvuaJW8Btfss=HH3fhgfRN&n&BpZt{V| z?T^BPVgk3uuf|MutNkOAfR=y<9eg!PyakT_<_KB(X+XzRmT}J(#_8=b;Qz{89kW02 ztg0L!1czgn_!b&*i3UzDANZ4+U?c`GI6oV{+uW@{D}nQB#C?V5PyTszCa|pO!1v+J zL6t+2`7tl!l>Ssoh|mN$94{Wj--g%}mJO7ja(oNyeYy_*Y4Z=t2@2&h9J#*<})>$s=#7 z$+*Anh-?@IwDq0+KjDTXU3_sFih=!#1@o=nOQ7$m`w!-1OANe44gf4%)4h$2>lXoB zif8teeseDU7yA9eo1863v);>Y(sQmT4qSDIZhxfFCsrZoxY2nE=u+I;SKSgS?Em$I zK%mey+q|)gqy36V-FTJ}FC;;;N-ipjsvFK)Thzs5ftCRJ2aEQb7+Z&ms~^&gZq zY%V0JJfoo;fLJb(}o zQ`;1C7}#o3Bvj|PoPI)vnTw7Jpu=irb<*R zxA_{V4mMcC3v=_GuCO|q6FK?h>zR3Ob9x(cgSu9VvZdYp@{i!c6q>qK}3Kz!S)h=z7_Kavj1Y;3fMPv)qAdk%5H&Y15Z;^9Ns-{awcMP2B$!uC| z`R%Ok>b^;k9N)9=-m`nC<$SQ<1i9%ktpY)f<}8cO%g}eWwJCrDg}ZoKh~N@5Zn&?h4edS8v|ru0@xcoG9F9w`Z{BDx-L-|wg1Uyd zjk39VDPR>T3r=won!6NFin_ zQ>W*wD@qbq-LBPUrL)VJ4D!_!aO7@l1a;Aj_=YdgIWG=q_14hG84K7B9)SME>x0nC z#y4rju>}V~>wCJUP_z6=(%(;o z{?IJ293u{bm_~79fV>s21ru)SWD5ih99t_jD>YL*X zK7?@?RE0RY)Sg)f_6C;vl{7cp26#JxvOBvu8SWL`+OK(l%6aCJd(spfIrieJ;*An7 z;oK^9%qWhKq2+=x4Jx7!Xno(={oLkd?jfZPU|ox28ae6WJuYweDHp>b@@5(xvYT7T zn%@(|NX$FZTXsb)~Lm%r5|Z2f(r zc%*B&;S`b}z-N^c2;_nzOT0aSI{407JgW@1;RAY%=uDgSc)glAxA8ONSUemPiMgoL zw?*iWEroOd)Y&!80pm&m#-6m0`feJ4qsnm?#sUgGmgiEJuhcvKOi`NDAK>5EQoTxB z^!fe3o0F88!cc@gj~wS2ty7SVu^KPg+8(K*v;k%ghq11_)BJEEM5iWN(qXz`t5Dp} z9SD&D1CR7Dfh^+$5GMOH7s7Tux9CA&9mydYSyIiFIVTF?Tm_ z!qeR4&cc{u1g~q{NeDvsVdIFCylr&bL$KYmgzo+>5|7KGm0CAX#V}LLQ{n7;f=wX7 zL4DWI@X_g=5fl6TNHHHi_hUIWGJP(L?B%&^E~|HZ)6PAUO${xMRp0YIbd~$^IrY-G zVqoD?|MrXM02*4o%HS-VUyXgZLl0SkQnISlv?cMse)ZR#qI;l3w=J-*Cktv-)?;$wjv-L#SnIhcEeJ?-!P-q0sZZd-n#E+ z;I4*tYYmu46I>^x?y<=Re=LNO!@=#rw&@|`*?M2_6xZ)K^+^|L;iUF6!74aPP*Hr3 zm__Tut=|Dy!y>i@KwK??=|P2wO@zYO>Q_+%7Bqg}h319UngbvBdMV%}mfA6mcj8;G zOig`~{f;c=AK}Ho5+xH@CzE1{Su}%+9qfZ2GCg z7;`e6VzMseR4N>2s)(15+N)9G@P{oj9xfublG0AQ_lM^q7^cEe?9hif7#4SI;dEUY z()T`nP-au!{7E_^Ovt$jHHvMC+?5MOc8I3*hVua>lnpS!f`MQ(Crni8tSC-x4HLQ%(0sZ1>h`Tz;^(dH zxOp>Q-N&HiTb%L(oVH~`jto0`x%~kqA~oPS6Y?sAa9((jFA^tZRX<>~JU5UU;LcXS z`~1W9t+~dQzzG_wIh_|~;@=C878BRN(d77E-n)Mw+cS`z;><5QVbPAjqPheHy;nDI z9!{m^4)eoUo3k|YCY!R*3iK!Z^qJ!2q=1JL~ngd(y$5kBtvd!@<(l35qdAWzD@S!_vPJA7Q?Vn^Z7~>U%v5fjHfT58Y z#a^l;b8t)MN@Gm38BijFgM0~zrJm#r+*j%tZ9FLdz*K2RV98)&<-@T5O#B5HbK(mJ zWV9Zhn4jpUMW!+7hVR~yW91CpkIf74xZ9uUoF?PF8lj-^k##EsU|<#Ohs5|i zlfEl1bwpGV94}&~a#^s5PS;wVYq0#R7a%_M`44)D2@S{7-@zHzlkeLd2a;=&)p+7^ zo@i_hpc5NxZ(5-eV-E^-8mPlxh0l#_bu)f_MO60TLjdNs?Pv7$V;3e?bMz_~bI5W3-AFg)npJeU~OyVH&8dkAFTx zMu>%b6Ng!`%p2dIS*k=mmvIxT+k?4Solq2roL{~-q=?sK>#`}UPjx4TfyH9yn2 z6JH3Ow6~WC4+-{Ni7chG3ps9BO+C?&c1cX$1sR#v$HwLB()Hz=ZZ+8kR*^f|0q1rs z?Oh5;D0%6`cZ|E>cT+G~Jd(yH!hMcUN%uX+$KfPM39HR=(`$TD^v~o;}$Kns&+$Qkt-MPuQ8v=P7$$=fR zM^3hcsWFt{*8J|p3B{18D(jPppeV^PCr^b|9{fa?Xf`D)Tl1y7tRP5b`;9f|T}!cI zQ*9bR;Py7j543*CFq^DeWnnqQMpGboK7>nDIf@a=iU`3QY&fICoSmKh+;Q^C#g}Fp zhRtb*=`NVT@Q-dk1sS}>Q2Vd*S`c#ZS?&2MpDjEiU+^o-rgJ*2tzJpF0&A>HoEeO-xerIJLu6m#V z9wlTE0s7KpwIPGU3HjGEZVl>Ip6YP6@!|CoE7-ch5OL^Za)Gy^%NJ^|x!0U`QwIl4 zjNo8fUn;aoM8*wbe?)U^borR@7W`iE&AicMnql&LLu`3|Zx@IW<6*gHQuGOPwY65y zuNVaIRvg*DOd~H)mo7ynY2=3pQx1$@Ri|b%H9r*=j*llNJ7!aH*cSt*AnubFB7>r+E2oD z3|nx2eBgzPS3;fgbQR+}99T$TA8Q;jjPIvDpd=!g;C$Cd*%B+shN1WEryrLC5u=w* z7NxoNh)(Q+xo<~jrwf-^-UYA)K;?p5xv&Z+k)aP-KqX}ahK?WFmNI@yo_yUCaCw>K z)K%!N70y10x?ar-SBOaCKYg`N@Vh{t_fBB_H)m8_Gt7FA1i&v7BeEY`V*}5@K2j!p z4djCTNK&`fV zUVF)Rjed6-;9+zcDzsiY=47|j7mc>{_NeGI;2~Ea2!0JnoBvpZmmZM8bg@Z%)jPxu zK^POrbeyD2zL3Itu<=Kh$(Fz(M# z5^r}V)b?PPmz7m#Nl9(+SrP#ZHGj(~{`#EI$rzg-WC$Sv*C3kJbWOCdPihY}WkjN? zV48;R^l&TszzZffXy}LpkT^(cOjvxk?H@tzP52Sn3cA&?--e3#(9lsaboH`v+R!4Jm=dzPE|jVY+xf z_`Ynq5W9m$`!=)RQi%W!2?cRD#wple6XQ-gDTlqD=+xfNi)3XDTXT)>kWd{vlZ|JF zc;JIXWlz<_p7K03KCO_B{slQAgRB705KZPTjf9M0INf*#Cj6KhdLH?y!i(@geaXbv ztSJ%KRGdf0F4DM@viP*LsB1l_O?87QTcOAi>zWK>m2k!?o8h2YWi5ir3c669u}{{1`7KJvA7;nNwZj>eBs&kf$VbQ#K9 zoS(Mffb}~3P;Z(2qm{->eB>in~@E@ZO7pdAEv=43r^28F0U0fju8Z-`ZU%$b~ z8hP$v;}Z~wp(!TciK@LcleXsT-9=)EXcatuXc*#2bCi-;_nv%F7?2IQ=IaX09TR9`OlKF%;({=h-I5<{_{1`Ew+B_6Z(bXI%{ z-mBDm%RYm5;Vx9`uZ#tmKTF1l+NjgJQ>7xb)-6bcqAmcwu(}}oj_mi}okb@# zaYs9L=BU<7$ggb>7hBXdW0<K?$ot8*+47dxh(UKX#v=1)J`-C1 zo{pCGO@?qk?k_+Qk}MsKsL$!G45AgV8@`E+c`5lZ9Bf_zWtf1d(xN@&(Aw(9iB(>j zh2||U&5waiGFBq%?xU^-dh~yO5m%R}EPKZ?9&`QLSj<<``_cxhT1l%va+x6VL7jVX zTX|)QzEpPIx=exVD5x)Tky~Dbpwv%07*VV&f<6!Akg`Zag2gP_5jcTW0i|o&P&j3; zH#VTP%>Ogcbrrz+Cg1i#|Gf`P=E<%-oPXe`T&}Gt5}TfMDIvV^dTj&Xh#R7BP>pvL zO|?oqK!4qXDq6<6n2~sPvScCa-m*dPp0&`#o(0&nflFIvImzZRm$Kr0M6+mTLVWU~ z_rW)~Qx0=qh5VP0k%5}CZ2r(NcGJ$`G$IgJMNvW(*M?iNpX-zo?stqLD8wBpGcT;9 z>s%E9fI|?SPhJ@Cb>ZYVe%xj7cKuMdD(|0QIenDsfk!UUyOp2dC=1ZPj}sfQr+JfO~;AIp~jzK-CHFq-@7pKVul=z3XTC8oEI z?Pla7vPRMd7xf=g-6;tKNbX=|s2+bLbumbof=uk>nn)L^!ISbQr_X)|8PPjJwqr#` z%z!A?kN12N#XzHQ^WYTf7-;KLC62$cIX}JWB4^A)-KK>`)oNW2+eMeL*`3!3Yzr}C zg#ev{+RQ%n>EeDoOzfIHWoGp#&KHR>wb;|^HV@|)0EV;LX-4iC#;uv`HMl=mGX^ZZ z-&gG&2(}T`Ay74exYjpuNZB-x!SVb@lmfOlcaM6xnqqx>*baS@G4Xbq>u>-Zif4Y;I%SwGIZNQ_%`Mh9vis>Z0b@;8wLOO2D?sy9LCS=Ntk<0z~)D8mUZ~2 z;GpKPz{mB2DK=u3R#vs<9;8HRj8~x=cs5-4dNkN@((fId8{ndTR)k(2^tmClW3N`( zK18={|0zC`$K!AAUDNSI>FJg+zHnIKc+Hf}AEp}e)?ix|>CWbAt{VkhwZ zBaBxe+V6RyKxI$k)Dc_O`Ie83kGFmgIkb(fI_V-;e{1ebWFG4XDF$NFe-Y7D2+QHI z-IC3b|T!nu!!rq<8RhXTADN{*=9zgG6@hZ)8c5R6*fjeoQYR_ zoa+p*v9xwo2p>W^iHbQGlYJQGf0)z$%<|DOu!A9TfWdh&Sy{Z< zbRWTJ_SRc%eON2%`{%C(Ky>MY7Zrg-BdNfY%T+A2Yo2SG7HX0FiOWJKRgz7!>{<5{ zxZo#kqmf>rG`0lfIkjVh(KN2EGKkg*-gQM31Jf60YN(k3x&&_{N8^v|c5B*3Gm@ya zD3t;MhpWeevAoSY&s9MYhej^;M9u#EM*b z=K#v;hO!Whp1Ft;`YgIAHa0S_F#aLnu~`n&&p5;t2MyifWqwQwx)6Soru6K^o*?S0 zOYlX8zCnPoPYC}B{(9M2r>RF=0H_(pb?i)}r46G;I331x>vKA}ER?taFz&}h6|h~P zhYSF2t8h{j!Xa!l1cE4Jx_O!0wTZsNh0sIyD*;=)Uj^bmM!zU$@w@x6V)4?ie3F6* zJPgp{eLUIv+gdmRy!HSIm9S-B0-O23cKsT&KnwpAUutSBm^HKemL<&j-{-xD>zoK4 zxY2Q+b&stGpDnlr&(!=@v=RYq){2nyJyKKX z;$l=6|8VK;`4uPY7xwL0fiK3HA}|2UV8mqhs;IB2H6li&h+hunmzel?M!8P?w57e; z$<12h<%TOj4kB5paC#QR8-Ik;^ZqddXx0bnE-+uH{JYEY{IB@% zlp{qO5JH(WtDnAdeOi!L<)FId2GTR z11kYEfTXzRwKXNaz?^13df8#NJ7AGtu;i9&TK#)?Wx$HMSjH!+{NlT&k0UaQpvu?{ zTPR#DijSdB)+GnSEVT60m-E@;Cr?B^z*In|mWbz)6A^WNN<7Dh8Z ztfD`^A+caj4fk%d&N+pF_JhDH`dq2!8f}I52J+iHJ*JXYbei@uBX05jtS`68yTDj_ zxa=|b;XIB!1E{HFS;ba$*Ev!cALIehmUJDK6To7^Q-`5amVm1`Yc`$@QKASE=~-!# z?E@x)fV?3{55oChs8hm$MjSS+>y!SU-enVFwu1s3dYqt3z48qvgd~`rYHK73*=G-@ zL-imvTxEN50N3!?)oy=fsZ`DU<0;-XYhMEoLK-ZI*_B$bB+*A9kFFs)yk-;GFKSQp zeSB^^r0RPIHG~@t0hyhuQ_D*IR67lJ>c;S(X4PvJ!Ow%!TVpWdYu>`p0}maIZB{U= z-wz4W^V3(6aP!w*aAy>Nn?!p3Qo-+JWX*l$zPfgsD~T z9;ucxnTlpwN1uPV?Sc^A&Wvm=<&CmNL#NMaI1-oN$G~?V72b?hPhHDL#dw}U5TK?15DFCHbcFB$kb4v}_VGROiU~e2QR*C6|7b~ZzYUoeGtdtf3XcoU@8g{) z(Py$ZaF1!efVy+li=LU5jV4g1D_#EhD(9f^v9z21-t9IL-F%3o!$}z@8L=?SY?mLLdg$;+I7{D+_AkL7-jRd56w7N+X#8!~g z@A`a<=zBk}QVvaoj(B;VF>X&FkAdP^%Yb)Y5f}%(mMja)nUODD<~@YiSVurEijIVQ zLHGm{WS6-5o$ewX!1*KH6-hP$=Px0F5UL1923sv0f9!#SiS4lmHeIPG&xnP~ zO9$y@Hh39_H;e=osN43Qrb7!I5)vZE%fQ7|Tj-{%yY?VlYB?|nGE9WFf+T5N-U_w{ zu&pcZ2j0LDA^vo17F7 z^NWF8l8vA82fT&2&P_(Z>oY2{2@Uy-(99Q}o&g4@KYH}&MBG6}Mn>Gl1nLWyE?ufI zwRE$>f*8&ZTE6)kFj?000pLstwwWI6RkW{D_dwh+{g@M<&u7?iCA1VMeO4GGM{Hq4 z`4h!8Sfg>mVEE8QnCe(FZLtjtcR?8(U}3zKU9Z!~V8W2B8#wh|b@n|$1}pC{`4P>I zy!cf$az6A#NeAlK!F8!xS_h9C63rPy_7rj-Y$UJcMMFBh1X$BUp;Gm77{;02!|$UI zv}d14$#j|dvI7!Xr~Awy1sQpLU(c6rucDtJV!lA=M}k#Z0AaI7iqZ;8MYhBG)N#i+ zWj`<-y<9I0X@?&7D{Cq(X|;@bu@HT94~iPGmGT#VRPwd6K&8E5@E(IS#W~Q#`c>o? zXk8)RBiPRhE^BObI}yvNeTFbiJ>9rLMsX$sOxY?yZVQ?8sz z8nY~-;!7Q=_^rzxycyR$=Sj}<=!9w085tQFM(?OF*QicN)oLI%?cxi4={}8HcJ`lX zSMorRqlf`Vbn5t-qw9dCj)(yqvBbT#6Q^|4c|nr;nc?lgSo%ToFBHkKZ_aaoW6NQ& zXO%9lqDYGB_(i`f9)Rc7Fj57Q+B$pgzeCiG04U+c1W7f>65T+-(Ign=DRIK2QTt?e zQ+=SnKYd(?<^6P+6WNq$dKBtjpEB$1ZzXFL3X@dJe`ij5b5sZofz6%QeEk`(TQ>mu z0ssv)C>?CFcI20#k;LM^cXNVImi=^icEsK_HlF}nBGc|oN^}=7=f}vrR)i)H3BQUj zM(KkjYDSo_$HDl(np7>3&eFFS05nZNI{xQ{b#(E|@*J%- zhNxr=B%)%h`<))dInD=z6LpYmAJk5pKMD8mT(4+bK+$%F*~mF^GI=Xl%Ha0;)m7U) z*7n+%e=3JmcxWD(9kn-t$Z|sqmK8Rr0yeM?qr3wCRXe^D^guu`)KxZv+jEMs5pYX| z?*wC;$Do?jdWY7^_6)>celnr_k|0ho;_jqXwy}}}%pv38`>QD>sT|%xNKqZlY*it5 zSZi`wVPwr?gr$}d-P9i9$W5cLjZZHhx6g0iN8@m9FU@gL?lrV05WxGnWMgH+!yLD@ zfKTxV(kz+9#o{a92nGk^qtgvWa9kIdM&M;K6>zqi#!X?lFUJMu8~-SK7UDuBQV9mA zGeG5k>YX_1a`?@^1s_qI>mVT@{ZYIdaCi{q0gBdAKeM(d|R!`83;Wn&BL8em} zD%u5_x8!%NG9W01Xz(@|OkbmTB3|>}QSOy|C7`Bm7hiWo?dg=+5&z}}A2D;2%>3_y_D8BiF+pfJMH zYMCJGdMqN$o)gA(ea>@u@lNP*g9bDRbS1s0@_|?9D9iAE!5-hU1|EfS)S3k*VuTncj%0@bDNddfzf3twrrZ(_xMSq|iQf_eH)w!$<@AFs zzR^MQRpx~#%L=~Ng(m3Z0BJN{*$_zC%_3-bVuaZ;L3=VBFX)Ey+eAa5A`<0AD9d;8 zx$eCr)eNaj^LEy6{X)=#kcYe+F0$HNS>lVcm6eqkCgo%hGb<}0Rgcj9qF#lHA|}eZy1RfYKY{P<5d*?2h8rmI2=$dCb zrHE(w(`Q@RO9?PMhgpRnQ)41ZCMm9y?&Q@cZQvP7W(W&<@;=P3?ZxZiWjQy$ws9GJ zcx9X#yz3;Eg#7 ztOYY)tBH-CsQZjX0(-wh+$%VR7#SDS)zw93k{A&|O+unk;N1}pNH=X(N~-fs`?z=i0t_}qiXcHEF3$I&Om5=@pD<;HF{uy1jX-j} z>$kie1eC*mjDofG-c|QcJ_@_K{7l2+dj0lsxPYY3T9x7c;!bn~6oM%0f(%;t!GTGx zHIZAf#VAhDM?N3Fs)GC!9RYzjAzA}&mmw3oSqtwhblPPky?^&4$GV?JS>57RJln%T z$Al4gN9YqcTGIszlNEX(q;*B=OTvV%>B2F)=TMqZ2{$40#3c)=2qO`HZK^5Tu&m z^+N^cbX_|Pxs00HU+e}6@Ud#K-Q|)Dp(Mp$xWGuQy6r#nB0=@qc{{swH9l32db_uN zp|~|9(P||9bhr7DK&FVf=f7qFbiR?)JM)WyQ5Z;0yoiqLln#72q5qVT1hG>QHzBy( zd%$EZKN!s#fwc$Tq81GWdD=+{CtCw>u*F{aLd$C`_j5zpl6&*yXVcDIqu&A(er2~an&|l@yE6{K~1FL-QYX)wU&ET z`J<%3`-m$pk%kV5mBepDYWU9%30T?|f~{u?zL~#}hsGc{7cQ&;X|es<=S-!RSwMoqncHFn3gMw|}!1yu(ot&c1Ye#aY3uJj@zs0lq0c zG1ScSg`1x3%rp}}3>@`EGWfy}U;4b?ZkY>*csO;-h!P-8OI`XQV$G)}8gT=CFH(8g zQb#T_WKlLrYWN6wNa2#_rW4ha<8t4&kFsDo)q1{A<%_mC8ViSy?XEps3+>zhBAD7eSK>%6m})w z6nHU~9-^#u{etk)Oy>woW&uR@Z!KYnHG>HkNQ|Eed#Gy_1RCARFWY>;3-&kRlxtqE z2I4KaR94Xo{uNTnVB<0}rLf&0cX{&4W%jptkAE01hT_X#?ePJ^^Qg{ssUTU9`Kg#Idi-yh8S zdNOhRsw-&)Xcq4S>iq+|w^s?3qEP)gA7g#8+w_POKZ0LyS4m;BY{;W*8#kb(bC0jQu#%$L1X^ZWfx0z)+SK+ zt_k+~6*8<9JswMB%d<*Xne@$!#e~|%gAKc2PwKtGm|wVH zZ+B(lvwrzP`Kv%{c#g*h>!kM4kl>c#i?9GQe;+R3__>|P14*6?%6iv3=|Y4tE|}2@ z4nMQhDAQ%-Cbu685~Y|WENKaq!_f1}-LvF)lxS9YzK+CzPQwFk`C#=CKNNB=BLD>@ z+U;8|@(vF*m(Y4fwZ00>`WVyKVgIYf6!1+(?tl2V|8!5;ed$HVqTPG`YO1^v7|UBB z2*5%gbkA!Z^ro~NubwGuFgf&+{JZ$s{xusH%w$l*nVrWMF~z&AHipH%`bhpAP$?sa zK^KXQC;b57{N-e+?C!_I$1@UmMt_+&G&o5<#v?YS5Nel8acdL%BcJl0y7*A0RHhR2 zZTEqH)uTe?aGL>bZ0!&VHWnKg!2m<<;4?Frd-gSrmHWh5F9IR*FAP}&W;9ej4_H@~ zC!kD+a3r@hU0L8QtmSq%xr)_>H_>qJ>3cl~tk_5Q@UEL2jImsG**i~bhVPQwWLUdj zmKpB{1S^m2Us@1m9K)Ok6?U@JG5YW;2kHcVy_V%}mo)&Vr~$CS_eRiKW5)~fGiP~t zc;f61sTvIEV&bJ6vx3;?@M#oP^8G1#f)zqT(-2P{5b^QuB4M?hLjK!8!LWf9D)jvP z?N(Y^BHLdbB@0DJ7V0{ema^-K2l3B)%HQ%twb{%As9|^}fA1Er8d9{e`khVY*(5Oe zP#SBpbBsk1l?YAf!kzGr32%QVPnTkV#|D7?F42QEd!!4%28yBL2Z~?*Ba!_NTm&$6 z@J7apJt}u?rkWCiI)LqNvTRyU->UwNnT3U}oeron1Jlp(*hKJ;L^W71)XFB@DeQxv;;$NhMRPc-V z1LAJc7Q@ry2`ugh_IN(U0u;aqDfPmG3gQe}%(qE%5YcglZ9bK$_p8j=el)cKxtbZq z!kHa|aER`Mq|ZAjNPjU@o817gOSC3v(h%0IhN7hwCdnqFk$&p2CVbhEuYn6l51ie% zu9$nDiA0BLOzGHETVkYXa$tRy>ZrTXZm5A@j2hnP$Z(()dkK}@Q1#2lSa-S@F{q}- z&*MZB$*X66XkR$IeMyUT@Gh&2{b~KpNritJ#o7&Q0f2TfIFL7dE;GXyqC%6$m$vUj z78-#wYy-;m(!6{GrL>X6ey@PIlaW|AS<06E6E6q$t)Ax@GjBP2_Zw7+hX}Da3P)gh z94h?MIRE9Njo?`KpY>L!aL&lyDsp>DwLaz?-p!Sop)vaXhKk zL!PK;5YLC)M-W+#iSZM#`H_yk zKikhqn>r@|rTF4A&M@@73?-}C8(XX7c0dNJV9%d^&Q+c=`uJF=3B>{UtX_i6;E~8w zr7q8=7c^vkEQ-DBr;}BrYh?Sd48cAC`~%?Q4`f!J3V{(x!Q&EwbgvM(1isqle+j$4!KJK6iGghu}W7yd&CS~OIy!z{b@vAO+4AbwFCc)+;$F6uoxSa zRwd#^Tu%AabeJClmEaYeIR#ucnnsAhC=mPZaR;>JN&8}rOCf>Woob#Sg+0$_k*Eom zaHm18*~ry20Utxjc9%&$`euqkQ3E9`buaQkn@A?U)@XRJoDDRth2>V-Xv}+Ga;Yf% zBxOmT&v}(hUsoE(mOhr@Q}DBP@_H{I9WP_FCUaKYpLwqvYUVJuwvbo1{;6kUUPxkM zA?*pSKR1GJ4&1TK{ZsHzlGD4WLHLtmsJHI-fAiw=y%+a9A>o{ETY6lS{;I94np!9# z-}&^!IkCzVIXXx$Wqxb`DU)vybb|o~?U_=zgLmFd-%D(NdgsFp6fJq-I=gT+zlIYA zK41Y6KeS~?@Q9QS1#D%Zw@KV{A7F0wG}JfZl$F82fF+Y?qc(j2m;HiqYr3wFA04k_ z4KZUdEfksV?QbYODxRA8M%h+&mORz;4J;f^*LZflk@*Cr#|e^H_PkS|SUbR)mK7;DB$uR*zUjp=o#jC6VHYVztp=)J!C9HN?sYMEEPQqLS# z%>AM<2}fcupG$B1M1jB!UdUb$egZmD?ghwJ-}U(goYN}7z$(6UxS zTV>8t@kbfftg3%|}R)io<_-Np_pA)oDZB1 zHT#+Ci!hoAX1VTWxmd66rW;PQ{TvY8J^)1Uvhv>N5+&zg;~BNRW~CNn-xrhdOM<5v zPZwB03SBhW)kZ&dz~f7{_{f;PpGK&ou!Jz2p1*n2=MF3BKZ#Lv@pWS`F+Qg{_UAgN z9?oN(HUh~H^m^rckW%v}Vi34G>%!GEpkRX9ieT^jc&nC5P%bwvUXj)2r}N%O@F^gc zO#9_l_%y^1mLXsONxC2^IR)8M4s0pam%_{KZjJt_Zycpe8WsMRMIjHUv~RA5jIF^V z9!(cj7jhY5qCeqU4K&hpQ+38@Y7cDV8Kb;&36|o)}C|gra%j63Z=~}Thluqno;os z0Mko0v+)(E6MzK7_+S}Qst;PMJ7~i$&6EWqlrQH>svzK1>hTT%s_hBLoX{ZeO<`FO z1_IN2?KEO>IWXaNrr?ksr{J9ja7NuKICxM*rU0;1b>@x0q0Qmw_v<3h$P|v1Bqt+d6ex^bo|gUM^Nk?6 zdARycw()ypXxF2ZpZybtHwD2!e=y1Wvm}20rG42K=`ED1Yp!z{Ty$t5<-MQ!`sML% zL?`1wv&Am1015PEsh?na+FXu(rwgD6UH#ofJQz7t{@1AN57BULQzr<1&=DIyt}UV> zKIBS6d_8LYz3lp+)M`6VryAs~OJ{JX-o=-Or<)!OCO?B%toUXZ1MnpPxp|5Q7P7?8 z2>0#VN1>+QOE45_mjqGC>61ck^LwwIjudfezrja3HGd)*3v?1iqoUP6*_}Ze353Rw zL%wU(@EEj^SqVgkYSR_t0waueFJloZm&~3w5cfY7?N{K?nO_lTP5UX59pu~n$V@2m0YSz#Pi|J+f{d*Yu@v$(kh@Pz*`H9^`ZU?AD}zg3FTM z8{su2{)0`q89|3gP%VPT2$Kk7`P#>z+1=|5ECAOv*+@_VcO>#@M9Y0w9l!9F^-C6_zpw;?WS1Cz)^QgZ0F zd+(*-&*H4L@i~aPCoe7NcdhDyYJ6yKOz2?wjffl;s=M1l{*@VX0zifMgL$Qx z_{4COnPXeA>(@CL9j+?xWOXvQsY4l)XgW~KHGV07t z!J2ir6oZrh{YqcvqeB$ZS6$L9Kw# z{pjz4IoP}43u+EpO=rHR*A=`rSw_JDbCUoKAHMj?#+i^Y)P7bRZ6U0dFi*)`8lDB4 zC@HjmGoX)i5x=PZ{_MEyu14*#I{Z{?;WW#=rln_6&jduY#dA#r8~Y4&O^^)78e}tb(eETJso&`S9@9g2^fn+HDLy{khMQCnn*RJByg&P3 zL+JHWcH^AOC!4~*!0~-)n|t$vKig$;MV${CwcGLF;#ujsZop*fA^m1^BTRMDCJU=f zJk&oh04aJUV+zGgecfL++NyDn~yoSC;={+_4G^ z2P$DAMg^u3$`s%48x|R400ZWT=0-E|7wK*D)@ZyahJnKX?2_epIX;BX~YDsxYIrB(YF#XJ;)@wz@<9 zXX8M0KrPw#_TuVJ`A$f@vRLQzcB*i>HvCnhpB_Ec2W2_wD-|_08BMWex1kr>e~d!} zHl_3%(~oeg!xG2i!jASGH_UA%y!>JYV@m{<$J5Y&|0}isOGZT|Q3uV6|6WxD{?!Py z9KXlR?~>qiA!_txpzM?LSVAZuTDUJ2Ju@R3d@4*gQUEU5b#L-^k8M50cZb9vNJ76( z7Q%lil?GXbJi#Pb*!>C&zrFFy`X z@b7dg#o_JDS+hJpMLBnvmO>3|)_vjoeLbfb1BtU#G|sKKCtnThEIhK)iRv2Fni zjDk?c@Kzes-#cB;0-{v&$m`vlM2^JL9DMMw~prj~QQ2BugC{wnIPpfp;_z@#eYl+|XT;6#ZVZ>YG zn4vV;ms_gyK;yf)4$i(z0crM&rZzw~I^-egx=-P8EL8lQl0VKLjv~pJ{j#uidfhQ> z^3p3y%|&m)7q=QW8@F4v18oKo8XoXv6{Jg&6tbp9nXy8RCu-z+b(~~)fiAc&)4{rS z^$(OkkGLwWcV4&|{!L&sq^>>m$hPu3R6SFxHkZ&|^pWY~-{%c|{pS}Pc3vDIvB1i7 zS*qC(_6ZC2t{!Nv`oTC((z<6SK^Uod4!hmy481^6HwYWxd|@&8m%M9v?9&MX+@c2_ z!rI1EXc)4!xF)n=P6+Fl>7xClP_w_?kS@rfPt)+uh=VbZ16x6EOHNwRt;Dq0eY7%^ zV6Vhm@A}M&J=E;u@>iNx7+f&#q{#+$kW!(e9GZ#zO-obYDtK*mhUL65?0NMSns&0Y z+-ru*^2zh-lreAF56v`tp}-E0ZLaMngg0 zJ-4WrzzkOEMA$7RH8n#|EUWqPA$#5vOk6r;BAYO^TNXCSjzhEt6b+6`7cZc>Nb!CM z#1E<|6n6%nNkrT8eF=3Vi+gjr7(Pc6Xg?#gI%ftiHBr{|*qJ-v$$ZHD_3PJF^>Ki# zSp^QO=c^EBEZz&>|F^uO1|>PJ%aFk84?sB+ca`^Z(qj)@>AD{!LMs@KMX0KfovZrj~kF5q$+}i5f~RO|tFZ3l3kvIf&vxt&$7@4GRY?qV#^TWFC%}gvT-V zxRTOPezgCX`|v%Q2mvipifYd7kunrBWYd1dz}R4gWzJE5w1C>`@Cy8EvP^p`t@ZmE zRvt6ycVg{Xzy%w=1Z^lZ57vCaJ0x;E24iRwBmiJ|bi;7A!4!A)EXklfn3~K&d3lUm z9VX4bung*Jg3a7$BM@;~0)`3%$j2mSjirRw79yZMyGu&NOD3efVh07wHNnh~|A0Hz z0Pd8ZTn~qKZ`6{&{3g*5hVNg^mZZ)&d|cZo5W3~_TEFDoOT^564xaS^ z&QAWzAkmxXzDx|pJS*IY9llUN3GLl|adfDltor74#vw`*J;AllOLCV4#7AYvN4>@w z3X*b1SW3C$Xm})he#)p^9E(4it}@Fh28Eatl3a$I+vDrVBO;o`$&Lrt+~?y_f!-56prq5uF2xVfb(O zVAr1N;Djw*>K^ua)%I$d%61mV^NxGQuWP6hNEq`k|iMhjN~xdge-q z&~`d;*4P(IQ!6hVu2ag|1dwC!Y}(ettdLb}Cbri|?|0aUh@jBm+6k+)WQVqu1BE|0 zR&_B@I(h?|R2~A|!pxggm|SKz4_nq(_Isc*%Q`!tW1%H6C{RJW$f$C)! z{RA3u4-*AIulscmxdV3Z_f<$)();5-?W)^z93<1_SPv5cycutT8ysBzgn#&62Ob zVh5BfZD{DezIn1|H%ZZIq{21O(Roy6*8QCC;g6|)IcjI%%6sJOLad+?si6+ zE1Yxz5#io5#ATzWsyV+1jmM4gmr$wdJkQ<9!s{dfzn?r$X*ywZ&%;SLEZXB-SJ9pC zi`~k;zaJ##MH04eyeKSmj3&wfs0egs!UG>b=Y(K6b=nZp&N%PJ{Z+gKLt-^VfvB(^|I?Wuiclf@lT4bR~hAquxj#7iA?e506i(bC_JnwN}pa zRH+kDs&dn+vc6uv!q?F4qIm;HyD#Aa!*+=)qP{1CA*N~YrVV*D=l$qduCCl{<`ODy zHxNMH+0eTiOTrITssD}ob6+Sv^`LCrUv4{Yx{y@HEO-CiqhxFTj<(YnZ=pi3NgNI(k-=b z@GOPKtHHyRWdlCZt$Ed2Is7|DIGcZ1G2ymQ&*G%AY0OKdvDX5e<fZ(A>Vb0DPxb5vzm2&pS7ADSf#XY>2I zRH9tetDy|uJH=jSM1u{weq_42xqX@XCI6>rjGDmW{R~%~fc#{3AIQ%??Av`n5R!sX z3*$Qp({hDlq6HE{bTJ0|umonMs~49YD0D3zNyf=ApRs2e&C~7i1PpyP?6%`;zPG7+ z)3BAk3|Tl=VE0 z^KEJ>Fo8xB>C8FVzYXLZkr&v~{&L__rymU=9gG$Fm|5xl){5KzIVJatTUUkybbH8L zFdIX}d$l@b(vUNI6Lz_2#r)1-Y4#1~tof|86`>YHgZ+bpl~4eCCj?hnkb^r9Yk&O5 zB#E5hZ~`+R%A5hr7u)K#rAGRMGF$xNW)z_>A>J0bPc#$;hR)E#Vb{RMF{aal?ZEG; zZgoDZ$plFZj$FeT2+kF($@#q}_Z&HWOY`1;gzP^Hp`_@q@c}SN#p+A|#pKvR*MX=; z8Wo8L6O}zzy#U%JCu!!nnPx@(?Vc-;+OL@WB}Zz*3%X?J-BacYKQ-vUFd4KPIv)9p zgF!~5L4^i~Wu&WQrEA3Jj-+dNnBGg|ZA|cOagdI`E1YD$cY4>je3cm@&eF$@k$?%34Ot0x9ootKQa<(dmYE& zVe7EmEY)PhWabxCLe*&TpiH8EcJgTb!f2nQ;yq>Fr;LNspM&LSeuh~>@wnChZ(QbL z<~SwM!xYM5w0sRrZW4W+jfnXejB}) z%e|cyl$1X!F#?nnqx-T4x04)vX=jLK6i)9HX?3Pj3sQ%Cemfp3nb5#mE9k7OUW{vf z|G@gT9PuEq9B;l6MZ^Oxet&4ZoeOW&uv;N@b>O@P*)mLtA@(*$*bH~6EgH3#7jh4_1 zv;rd$rBWMzusCX1552BJ@rWNV<(BtiU@xWNUkuak>kc!$u9QWly{>nDHy;B=?>-wS zPaQa8(`xx7jq)ZgslR@9`R~LC09)`dSLU+Q{I{Y`*{1vcA2FZ`o$oGt_L2Y$v(7;Z+_(BI z!gf|3T6CI&g1==_vz{d#yvGp(1aQ-_#H{H4FPqOsqy#OOmg1{16y;A6;F3@6KYrnC z{k^TUzQxt&Avk6Ib5Co5T~?4PU;XVU2|w-X08VD%q*kvF4=+yR0p{QjJZ<4bb^dyS z6yWkkg)YQMs^WIIGcf2PgE!|Jc0cbbCf+r*Bv?BXUp`ONBF;?HGVY47klA}GjCndP z6fwgE4TB@wV|EME>LGAQzAhBG}5)Y3)}S zF=HffBkJ?49$}{Pe;rb*A^^dkMy<-D0{XoNPu+}vuM`$_%^jZ;!4V&%2{xS*$KmX0 zxt=yeA8GeX7jrt7mR=d@E-KQR$ZIv`Dtw2p6_Ws zyxz}I`xRiRQ$q$T@+d^X>d>zRV$Y6kNF!ic(lN%Xo4iB+AGp!5NL3BoYfa(!RsEzK zhEWSN8neCI`~>iRJ}w-hD=eVeqG-_P2LviQJaD@~BV=B=^Y95aCXsCWw)Ktvb{5fj zK-_-rn8J8#Z1lGJgWBNZW9hfb?KX)iociw|Oxmx^lylYv8Giu8X12Y1?b3rdhMH!e zt7&;$4S^ZwbLg@8C%L18?E&41gx@*ZS2AS#BBS32*&8gzr2{mKhsoe?_U6-RV(TuE z7Og)p7<(hDu9dF)W(N~N_mLYDSE1XdtAS8S_Fo-h8hf)kf(YAU^i?k$i@zvab&4yOtSB+EM!Q&UMDW^Xq zKXqd2uXjQF6S)w~ejmX%Q=p9*Z!oEx_q@x&I2L(R*Hp6LVU5_#`+F4*dQS|3)*NuZ zCd$)&ZuoPbv#5vI_8m)8Bk4OW^np- z+iT@AMg5%oeDVdr)UU6$@1<6A+y@8jL9snEzjA>O3QvWfa7H}6i;~o1 zu++dfNsn%ngE7Lwoh`FNfj-xV11V)^ZoKvcj>Va;C&sE;jAwipq`aktS1;!4_uE0F z8UQlP2$?3-_(Yb?$CTDS$STWO8O82^3CAZE!s4nj&$0#wB=jo&!+ zS07D(P?3@ERsCOOs!zMjLJiKiC-WBKIDJa#EcyLhrZvjX+8wk$X8}FbAqvW!qd*kN ztTC!=m5`x94a+~E2x)`UG;?J&rs*C$i0hsoq5cdCm<^)1BRGKCTKxxX3#m9o< z99`2p^uy zJ>zJ?RgR&gyh(4+wF$c4>A97MuOPz0&R&-^D5uBR#h1;jzevBgR@qAYK;}k49t`oa z?Zg9xgH(Z}hIk+&4Bc- z9_A9+K5QcU_Q*HqY9$h&ngRfl0d}WFx1(l~;qwY#q7 zuDQ;|BTVPc{jL%I1ggx~VB;&}PXB1iagJ6ItE->fukpc~5GwGfpXFzeck zGQ+23WMpVZN7xt&?9?Dako*6p}6IO$?CKWq7t-K zh)I=?r*HDF?fH819nX=sl@%4l{%Vt}(0ZE!BKelzs10eLzX5!xuiT3J`xMLou2L!e zUfl(wr@gmGkf}Vx{%+jm=o8h%JiV48I%N^-#8x5GFyy~-x(_y?X{{W-u#=`m>^7`v zD5M46JB0iod7W`%)PiFuI>^WyyhMbRE2w^nptUj=OnR>!v|Y=QEFyqvaLvZ3b%VEN zywUe3rSGtCAF_Wk?zs{Ok^hec0}D{^+Ae@HEZaZu@bsrc)tocUIv=Dwmrf(j!w>8; zkHCwMW(s}xADl)$3$SJOsg=9K?R6XZ^_>5b1}6ddVtzCc13B==%l-q)%CDLSuJhkt z3<&C>0>k$(Yi%2Eds{m23c;Q$+hNpQJj7uG8h?%2EplS~jWnupIT&!A1%>_eVY;pp z*>UR+oHLB1h2>DzB3e*`tUnbKuxF=L{Xqb~J{_2Zc{tXGM;NxHd+nvM=P>%h;J^&y z7hP82L6ZG$Fe{xyvgj(w-GTm%cJRMD1|i) z&ui4a=eZ;a;>^l}0?*E@hZQV8_WzK(EWAcB^4?mO{7S)Nxk!nO_g+jVw1_!}BV-Rh zl~Ev~eb4;jBEjLqCtG}uUB5=JcI1HUX%@W&6`3}IJ>s8fU(mjPF%Xu<=YM~yHn6ob zuoc4<=1?#@KIv=bIONbIU{bd_Nqjo|DhJdBv!Xs4*i^#g-QR6xW3-JjbSUxQS^eolwX04wLH9y6-*AKwU9D(Di1%D(2uLD+~EA&j+HgkCs(6Rk^)Qe>fj3@=a!N@-@Af zmE}la{^4+S#YM6Ke_bQ4knTL**CDf7z z{AxZ}U~?-eXBio)r0SXh@r47Bo@W!*T@LfAfr56K6E^bLmG1S`uoVC}c%tr({w0v1 zJp`dnLOKG5#3ka5QL9wTQ~M>dl$Tz#7r&$_q5Akv@RyL6L!Cip*Wk+COkHl~_WWFh z$&pSrcB2r58F$8&i$k0Xts>DSR0DSm@3IO@UPv+oout%pf(~eCR~J$=}m5{CvQkRaCsR<~$W z1PyddiwB(^z^*be?9zsuSIm3i*uL#v8RVghe<8Ph%AxvufuN;&GpZ~N4|=ku#~vdk z!T-`YGTViR_@F$vUID(|f4WwcJwDC36HQsS6P4+~mCK4?;P(^f+#9fv z5qU|lT2!EsJZhmm%xInPu=-Uw)q(Qudbr%Wz@X({j5^(LCjVRt-iR^Zfc=G?e4f62 z`-zS*hg+)3<|PNL#DapZ1J;MM(FOc@_8qoZL;_5fsbz~seV~}g5PVes64u%%fR^)t zis|7e7{us-uCUdk^(jS6h~+z^@nG2Hxwx|d zTH;jp3;|}}U2gCG>ucyf8R~I}CUycvxhag;UGpQdz6V0oResZ$A`6Hw1Cn;$8I$Sf zHdlpABb|Y$=dNR500RejzTJU_TlYSV+@s5vo%@mrF%EYu#GcSu_Z_ab0pZ*L_#f@R z9F^mbp1ep*YyQPOI4qc4U_|E8oixdZkux%MIHMN@-hDfb`UlX>#r(c`Londimv-Tm zJS}%DLDb&$J}(Xe^9#g-E59qxM9D>vWcHEtKBaV~`iKPp`*VrTfA0eA&o_@N^gGoE z%u@YS!ByiH|KQd=NSk=OEQjwQ_($nD##FoV$JK7W)x;-lKN#9YLBH6SDBHp5nqgF~ z3l?ASJ2_K9A=Ur5JaPNw!Q6^T&)MX$Uz7NCg%%L*tQs#g( z?TSN4LR6+?`;}YOxP-tIpr`gOowk*n#aU#c4MLNn?VQkp1Em1O$Ql$9QHXxM0RzH+ zoQBXH^IHMlftGu+vCO}^0^y%77QYeA-yfh9qLBY7)K_0tsxm{Ml&^omKmyE>I4I!2 z)4=#JN{V-PNcA&u<0ZhLt8wK$cTDGP7B`%+|*aDpyhe(O$Kc%7zdkp!gv?$la~xWQKu0z;xb2_7RqRD zGWng@&Guw3UZaaEuS;-SA`D|4`7?5~-VvMkWN)gqpZL$nnWte7eq3Er@h*?~Dr;?V zmH5{rCXkn)K+A)sC#wIB{N?CM!7)afSF8Fin3OFysEzcyz@{;)%mWs6!l!PS|4h%M zdY&=MgpkBby)*}3p^*L>VQu3$SxDFc=RQIGp95J#;yJRT&lyX~lj!<|q9M^0kyXuA z5EQ!%S7r$T7~n;e+G0WWoc-gtUK;--ocftBlt`GFp4S=MrxR)ueb=;tK9f7>k`q=v zG~PL$IvAII(-ArFfV+myELrUCN{eEVJ#(z@U8yYtoc-+NFTiW?s3`HP*Ky)KWX=)u zGo3{)LWMie^a5X~kv%qVD3){uScp>7XJ3PI<6lhE)7+VqB=;&Vp!il_?Ayswvy2;0 z9=zgUREkS$Xfr+5R2i{>BHvs;ykQtXCJ5jAN09stAh8jGYa6GAHxs$@F`|{V`67Qv zo5uy~k%R-R`wYv`lNz|CVR{Z%gYwK9?7mw%44Zn8?+RPdifQ`?sbJwe_WH!jkUl$8 zfgUuL=v4>wUu4{K><<6+oxDaAz#8s?f$f%*b5qW$x#nI_qXzN^oMT}OkUw!=)tG)R zKKu#h*18(Ld7-tP>q=4Al#0|_0auIp-~yT{e|EK1i9d!bxskW>C>^H)C=_VRPpG?> zV>8ueSCA!|5Xts+1rg{z6zj4=Q2`YwWh{vp2>IbIfEpp;+}-T&memX)ys+c~O;%FRw|-tsTRz=+qW{T3A*SS_wcNQM z{G1$$CXm~{rWuT9YNng+NbpI zy(}up-tQ1LU2Uh4G;uTb6x(g)KnmhP>vasVnHy|;=i*PW7%C4@gL~?n4JYJf)lDue zTn0b5t1?o1ztEDYICq2k1;KO21lQ02k1ePFP&;S9v2kOPL4noZ4YQ!)h4dAPG)%p8 zPD=jDc`twI%wNyt&O|M#n&aYqbW{ea{QbeYuKZgY^Ix^_U1uD$24cA69LrH0bBd40 zQujw$dsIw+#FiYf5EX)x@}-r~JpV*@s7=^6o{yH5Jo@?V+^tf_u2X)Wxe(x!EaLG_ zorkn(h}NhM)a{r0UhkOy6R8#VXBg$pU30;dIC;#r0xMxy6rhME^j`c%9DJ3vvk+~L#hbl}Ih)g8=SgOh;H zH_Y$buva0M!^%ixidh23A{eAYud74&o1eicWTa-Z<*>vW>HV;ErjeDQU zAPYkGZ39lXhUEtJFA-MmE|E%|CeDc6ZaO6|nXHW$ zie(XEbZV&(teKE?L#R1c_9}=~9{f)nfNmkU->d24ZD}OnA{wExz{ys`yta`-<-+2T zyUd*p2Sw$|=q;ObFp~`7eM1ZRq(iZs{1c%OmZz3_8Nc?VFOaF#LAEH>|F|9?lMaCB8wm zs8ni!Bk{7+mrDa;!;@n0tvC<;1ru}B-5gUSDS4Ye1Q&*u>}fF~aDa6GviX1t4X}-b z`m2U;9yj+}@)DSVg&Mt0?@LBGSH1pan6h4F%NxJY-X;|!3?+Y~h)&HrdK&v_=ZOF* zUH~!-19?q85J$=VrFwsg4`2H%SNg*rS8z1vEWnEZXJr~K0GM1Tv+xtvC_8@Z&mSLq z1R2-PI!;x3@K~N27pVCd=45UAE@F~&fHhn`PL|*{!J%&^Uk+Eg0Y%tUnTcfUa-OF3 zJW(xvN)AwR2}j=#C{)LsUTlN60yhgXw-WYOe4&oG15*R&Okz&4*$8iX3bWbNxpm+N zH@)pl@Ee-Nh|M1tqyCXI6v`@S`&&#f*t$DLIxM}s%+WJr@>3nz96~G~z-i*NTZ;uH z(tcE$41borghx(sj9bswpfk5|Se6+3LHztR2}%Lcun zu23lkdu5TN{}aDQLv^+=*e@0wxxIs61riOrU@9h%zW{l?tgr-mqbsV?H&Go-7$9|E zo|1&S51pCUf^D&`VQYj*oUz_!x z1qbfR%nm4K#GnB{z>o3LtU}{Qp=8h+k*bUwZ;WtcmUO77`fF8&Gc7?r>EPLBHGpP; zmG4;_!tbUWe6a5cLP9EZfG6?wW2YI*rE~1u*kqR65bdV!w9A(|GAc%M`!UTL9*r`E z|MIcCHq#hjYn{J2yHv5; z05mdIn>xg;p(WZ7y+1#2a-*L05L$XDtMXv%l_WS$@vN432ACmp-(WWJJ*8jK?G>MMzfJ~Hn~iI`2x2)H zl^%WMY05oM9J*jQX6tCNHIq^4`h4;sUCPUzYleX#wCN(%v!vsb-14r*b zY#AQI9Z|~AQw#(O<)rsDMc=-i|*UY?fLcJm2TpSkH#J>oy_4r`F0-uB~)(bUv!$y9{(n*`QG$=ym-!A zQKhScty)mRpL$XcJGVB`#_0$VHkjjoM#%3KK2(1I7|a-UVK4)3n`KEbk~x7GOSQ&1 z$U;2tIq)Vb@a=*%u%EhaJ%VjahXghX^#wW@njBPYL(K$ggL(fkfvOCTJbNOuy+7iO z5J@C^5uZs~UqG8|c_bLr`^N^_(~bG(-JbX}+9(O85PI!*q2fVC@BOfXEd{lG8{^gS z)T;mXeW%ZQzR~dbay#iRMi*4Orz3&hcXG@)9E!DiGXMi)`S)St*`D3z@jVejQqR-h zBb$uJhwY)t(E9#-nP-WQO&E_)&;P{S!h;&NgN-3U_pGRN&afxL_l{yP+!-kDCCDck zq%x3=gr56lh!(I4uVKbNm3d1)TSD&z-|Wij50V647mTwkO+C4Or>pVwhH$d@uh;t% z%@|J7CjrJ&M1V_T)tf)kK!d<5n1-~Fo&nejN!UPTtc|cm&>h{zv759C0>p|a8h|yn zAS88tV>iLtT(U!)6_`7^V z&5)x!m;Y7XkdoG~vDP=n9_ks<`u-wVz9Z{zz%-&e;M7(rPaBE{S-=n7Q2&zK2}+!B zZK%PR7KDI{^{)Qz@A8|d=QKQ)=qj^?0L#965E$?JEYj@1%@W%iuIO$UWEP9T$-03( z37y;H5j;?Dg~STq$UFs7HLo1AA>~Ili2nvB0w`d}`kXY9U6>yYn=cx5v}C%)KI315 ztW*WT_#t{7$Zx40M2_uXpOgn8PI1st7?h_uuG)fSNGgI_V`X5=Uj3VLAjs11cQP0$ zZ`q`u!%u;5Ws3(kAz~}>#xdBCepnd)+u|{zgC`Z~JTb`DmtXR^Br4S@7$c`i5QmF( zsfkS$&Y$mFO|>d>jQcU+8e%Z6eBpffRoZeRh9?*Z565(Dx5fd^3&ab@tHqlHMq?um z(NWJu93Jg!kFvcZa9 zcT`~6WY%E5p$&8l=7+V?_o{;P)M&;K)o}l3o`FTx@g#v z(fZb)tpnpwu>yZz1q~v+GqVl3#@pMTJ zM!_k{M^m2b&C9F58L2(uA7hs>Yqdd&rIc<~6(00?iCK4qDT8l);S}kMHaYTjajPCY zn9~ur`WHZI@y$WgCZd%0e$Z`yKvZ0#B3qg+b1-Jg*A;JJHP8-2|6WlhllR4P`1@t# zCP+X)P!Th`gsP!DjoK|ZEa|RUs~wCOF2!3XNfj3gH2=MzqZ|0lP@%ct(`>~=Yc$QBTUdz(sm`Hor7_x-obJa`6 z&;Z*OpLxhMnvvw&ndYB8VFWmuxL8$F91Xo6yL#79m*c}wpnsdO^-wkH^@J1FTZ_DR z*n28uh9x+n(uCxDi=P4IH}S$QsdmkFLDx%G1!}D6h*fDqz?& z^X~-SNoGne#@@ zeKS|efiH}=IkQOY^ddMAs5)t00q-NJCh8`Av!E*M+WBekti0=CY4#jZZPq|-Qw*$mYy z>vK8~sEzP0HW_!~nhMeq@uMsFE^%7W?nt%ldS~*frZnz)+&l^{7A!NtwrbQj z1>cS=it}r8rL=e;FM4S{vMF>)sF6QNC^d&MgOt5(IzmeEcD$^faNLx9 zd9S(9oGs5(=Utz+Sk`lflifw?e%{TlOJ1)iu)K6L5l3Kd zQXzSMqd%^w5@qnj!q>7?R@-L=gdC1AI2?I=*9w zNUNuRFXM^p$Q!}~C%!>ojHP}qu1+#i*pW)8TW_84u0gOa7$YAKKHZZFy-Ez~ciS!< z`c53fj#WI4jL^Y%jzGc7sq^(IL1l2|8CTz%{+D&0$ca63u6Hi4iKFOHK z!Ad}E#q&Bci2L+?y;CgF1I{Z$fL6`D-=Uk&4l|y*p(hzePIq-T#38Vm2Go%|`;?pm zNRa#lfMbI`QFSmfN6y$8eKLE_&J83GAB%t{Zjkl(W+PM=mHVQu)4dXxcSJQ&eOs2pwG+lPkAD~J`$GI%PEiaSG zwo`&lwsf4&*i(EsR{PYj7?$eX~uGt4v2vmATSiOZxYlg{RrD>Glp z`v>S7_Q$}-uiWp}#w|rAbY+Y?%jKXqLDr?Pp7!n|uZLsh-D{PDG3sAsCy>P+@^i_OdOn*A z#3Zy*FC!!HaCOQuUgwFQ$N~q9HPHb6B$Q2BUte=(I+7r2^k%x`xLphp7(y3?lUQUa zFQC%NruPm9;EeIozd_%hKp?{o^`i1!gEXqTrh^`}5$!$MR3| zL9jFcsSoGDGtLOssAzbGdwVYgs;kc>1GfW+hz1@@0GIL4;^HNFYUE;$ptXhhVHcMUr}W~g z-bcA348T5>&NsN=nTb0Qr_3tl0E2-eFaOxZ6zwB2~at<;k&FdS+ zuMrUCD6h7O*tQZPDnh*~G$LZKvGY(~A833y(mA4&EVf7TzNR%+WLmIzq#o{poW-5J z(by1{FJHbisf3yxHdfZEogcbk3h$8E_ zPX6`-iFl(81OTDaj=TtD2VRXaAZ2#f1lt91yp7gU)<sty$1IT+n37oJYw_-h2vT3{T!K^gIxnI@bg5jMBTdgcREW^ZTTZ&$#s7U!o5f5&{F^sB+tB6Mw zMFEUXw#H@~zj_bZG*^H;?47uFOmsE&I*^P@IW=5afWj*T1CRkwB(ih!WeDX$Qjsff zpq|zO#}zD}?z+bS7tpAAjA#H1WU?56%W*Eb;E!@#pQbSQuC;7=-Ts0)B0$*n^-ehZ zru-AUA^gDLSjWLBz=#Sk2iTfcnHk9h+1$E^0KCQvCzPak(Uun^>oAOyB^gXmO1T1z za9#deKG@eMd9VP|Jd&%x?GWZ@$;kI)l>{Tmcg<7mnTcLg!f~%3X)s@<7#Vw4G-y^ z)+%I?s=mJDA5?YK0*FrU-;usT9o%W8gYj;t4BzT}!LkZ3%xNv>H5I15^9xQ}Ut4{fs<02ljT(J2C%axiuf_;c!RGt0H8QUzJ-4xktx^0#p-wdW+<{o?E#T60Ibh3(~)(;k$u_AKe)j2G3TjXemskM-q1n@zsGI1bSkY(2YbPW%hSvYK!@{-vTlHz@D2T$ZA624mi=>i-K%V@d@h(fDbFSu?IWG0TY0o0}bSVD46N{f!;`!9YpC`O}X|QjD1(IT?tQ60$Y`8)xFkwRu?$ zYP$|u*xCJ>n!l;uide zRCBJ^OqST(HLx6E)Aj`BB!IZN{1N4SZhG#SK^cZ1M$t>++5chfy#uk{-~aIvWi%8e zrQEU`i0si_*)k(#7ZS2p2;G%YAuA(9GPAR98I|nFE<4GlQg-QkJ#YJb&Uyd-`TcWF zr^9`}UeD+Cyq?#1T#v`zSQSQBO!dqDPxKJhF6K8LT`95o8FLM0yxn|3Z z3Miy}r+jKlThayA=krjNE!*4v5N0ly&mFCXV!|7U)mWf6Qq%-WVuaI-VTF$t{9%Pw zzz>A{25`{F!iDKL5^27h?Zo#-Zokg~me0SDb-PH+ z;wP2A-al5i0V*Z*aHIW%I&H;t#eU%3T}aX-pz}ihp1vAGBnZ&FYoY@Dn-AGnR6Pkf zXaf;y;FiOAt5x~~$O(WTkOHENN34@@E@hfx<`QBuT3wJ(A*bsP3U=M%SIJ~#&Rm1g z+qL8MWz$hrRqc1W6i;k3J<)Xx1pE@wZJ$=LubkrY=*}-GN`z63b!Y<*gf=jUVWR@H zK_aJTbBKfT%|y;nm{_8Bf5zqEMT$hC07Wnm9qXt*ruOvaAx|S+q36o$ioHt*r&0yP6;9QDpzQ_KBoBjU$H?sAKDaH@^jX5jD@fI^|0>GK9#S0lp`(ueF z#rOa`dX=YT`wEpS(LRW?;EIs}U9l|_f}LFw#;L+ljl?$AZLFbzh0o|uG$&n<0^ipw z)s(%YKwioKpRx0EFOu4!i~!Hkn}Q^%BS;WG*5y|>Gk>~o)l!C{I%<=mPI0?QOUTtL zt`TmFbD#-G(XodHv@&4MCrgxmfXdz7-5Tb-+Jo}}1C8kK#^<;qET2Rn0hDaxrZ-A2 zSb0G|L7t=Nmx^M0%({9RF=eF|nAX5G*Y3>4r#oJLPLq$rD;0i8+$P)~6!RHdi<)#d z-q5aUmiigt*kBU^p@lp#4{yl->$!WAg0OsAejoYiC-E>ikDS>~AX~E`-DZKdrI)zb zd?%Fl*oQ5F$`hbyk5_PL&^7CFFoZtY8T(;V-J{s(JH&WT@?7e@dxtryMhoC<8s8N} zIz>>h9>^)bA?$D0iOJ=6 z1Ue_A`}F7f6TYPgVLdGU4vK^$ctye7vPxkRoEa2itcpMQ9YZR7C4c{X%W3ft4YdJ| zQQ1d_MQ>DOIR3n*-0?De7WHG$bY*Uwr;B;S9SM+-@;j+tUJ11jvx@8O_hCpx7K}>% zSh&;Dw-Rm&0@SYZJoA<_n0?3;Prl#2+QlRF!8SvWl>{ov4!|p4aB#li4V3pe;8T7+ zhtXlez12$zroG%FAq7?{1Q81;{X z8%(n0eCE2B5C`QEh30EPKQNSLu#^o?btOTWXz2&;DMO*KYmSR0lqv<^)UySpb7F{R z9lF_*@h6wbKuZC7@4JOE0c~pu80Rcu;>Rzw6hzOy!oX?W^z?@=b@-Js4tl81x3|Bc zt3wQv**WzfX4CC7dSaaSitwjYO@2gBo@3~4$BIedlxm5NnI#!UV!hyx+6BKbtzY_C z#LT0ss{z4_1+3i7Q5n;&)*Q7zoH?To58SQV7Sq&jL){i>ci1)Zf}68tqYUd@ zi^%v=pkavwzRwBVBrsaCasdbyT4W9~o`@h=6^d>BqMYdDN0UG=j2$Lbci=U37kFIS z=o0kmC^%nX07*5|@?o?*9e8>K0bq9{e8*23Z*%*ag5_>h?xCprtZ2|%`3(O?r+gYY z<*Wq2t1mmQ&b5-7UpSV!4R1qL_1pZ8U|9H9Yv{{r{p#!ycA2-enm?&NlWGNE17wO$ z@F`C8b?9x10s8X8C)fi5*KIgZ?5hG)Tf{kv&-j*K+BOk)F!V0J{1ZRilX-+lTVY7Y zOm3$sSH!&_qKZW%q1>f?DsV_jZkuZD>Y9*YmgCQ?s6Xdj0Yp5%Dsk(FJd4m?-!;Fw_a;E33_wK2 z|9}fbZhK1Z^ks`$?cc(HJd3i+{I8SOilH0J1anfYt-?zPoK_vGUor`t5^AC#_2f@R z#Zn0&_#s=z;^{U3pbvKzWTm@&SU;>oDgXc+E51GuLMMF$cKW@RG5OGYOg9{SSA21MKX;cN?NOM~i%^ z#10{E2AYIJ*CvV=%r6(;D%fpf4>}JuFg>1@+EON{v;}%)PeI^NT9}VY5l(BWHLU!T zOG3PBI9d!PJ1nmM6NRmJvHi%Axjvx9tQ=QLSzKb8-8OfaoM9WLp9eO;5@6U|JlF(M zqA)-546I`3RPUkHA9u#VRZ$I2>&Vw=`}c@|gb*}Xxmytq9n9cyB8D=1tt*0#Eo$mj z>0>@rOQefb}-sFCWvK14{NPK zOHJetSQV;INrrkWZUitGG5M{BsD(f(Bi3!m6K3DLQ|uN-CB<(7nhTrqs?NjH*IlwfZ(rJ?SjGN5tnyO(_ zcm(&MfcBn`bBMtcwTC-;0A?Bs%vLZ@VEe{+CMv@{CSpV_cR7ynxc@0ogPfe)01f#( zJ4>S2{(NIN1~-duQO~xT)c?#j!6pgh`Kg+mE#-+%xlf9}V*bkq8{qFaBi zLrI(?LtkM!t6IZh+8(KELe3rz{-N`C{wlGhS8OSCR2d;@vtX*QUoL`w5y<6o7mp}@ z^?Rjn{UvnL@ipY=oIuiu%e{BYWjIVDs3|16bqY;zeFZ_#!YMwLdIilEDU*!ttD22) z)o&s0J6x;$+T*?peXVi4s&s|fcV=y=M_rlHPs0kCy4Ypfayv3)OzsZ1%NCXURJ__7 zc)_A(?$5{Wnfux|%jLVONDM~xbNuC{E2`vk1b@8lxvYJ2x-+ZgdPgp|U6vV}0V~A0 zH_oyVw3*RO;}wPtYQz7DkOgJ*kzn%s?o3O36ckFErn6j7;m(#cE!?^V_zXst)Q);uTMb6Lg}1>9rX8-n4B@#yr@r_T+q?`rG3X30O}H4~IsT%c%Sf(f z#dK~r5V4HC2{`c`!M@Ck^)xf|aFB@@cK8_0L;|wC@wwElj%i>+WB_iCH@`z+e4Ea7 z_H!-@dm6D%E|S8yI$16<);_tFk0 ziza-Rjpx-Ik1woG;!JmNYw*!+8NB9sIN?c?Pt*9xNHyz&3cU9kP87Ia&;bI-<mq_NNf+>YyW&5$|t^Gswzm#T_2Z^(-gS2ek;E`Mek96Pj@HGng;>~>@va_mf)ia zxi&i3ggE9b#vD*Kl+yd|#hF_j?9VW7Zi#I-`IMA-erzGyuE3}sA8Fx~dmwdO${~Ha z{1mI5OF?GyT--26@;UQ!y!gyS->!S!o%<3;Yka~xT;JFOg;&OV(y7(^`UjQQyha{9 z4-87}Qd{678~E5^;OvnzapP7eP8V`&|Q*aOP_V|GeO|#YUx%^(JF7KYIIfeHOKjdvYGw4)j#x@88Rw2R@`HXE7 zXPw;u>x}f(o~1r|vvNAnp-RTWa6s5r0oDDCbD7hT@>Lc&={#hTLnA8VDYq=18#LzS zW&{P=pYhKu`qnf(%$FF@sGXFD&rE(eK9Y678sCGjOEyZt1_XvT{V^Z4J&>pVCqI?7 zaujM^8@!o4kurcx4%%>3K+Pj2WUdH;L4F%NH0@a#>Xwjxh#tXW9X-PKeTZ`LD|B9X zZW7NoLW$rQpiom6W9aJ#oTX?W0*B3HNA0TQOkx**a_k`}?&fw4EYKuJ6#vR5@5wTH zgmkJvJoMVAQvcJ1sZmw{mH2k%?eXZ&ibjKo<0%Ev8i8;9>4h0(HC&CXxeO5~>jX_(fMt}>PmB91bvR$8dg-aD(e41o=4=0;^W1f_I_ zEO8Uj70QpGe*(-;4BO*oFTP#z?D`9Hi+wf5QXzj9W@@0s*AHw=4s>#UG^$A|5&(^g z8|RJ^-XrMpIZSI7qzR7vg*YZ)m&8a;yQm5Dv#p`8EMcOx-v8s+NXsCWV<~<1W;-6M zfG@CbO5-TIk~18CXjEfrEdF9ke8-3T+CKR2o-f``h}qSRKQ~Hl$PF0*7Rt^0la&qQ z?g*R^=FQ!qnPmDn#3`SQSCB;a%~@N+u3*U4ybo$TIqW9}cwbsvs(RtK!RP zm#A&iJRzv?=+;(gI14X8sij0)?DzL`30H`n7NO-D49u>k3$=sh;tIN4n(mz1c-6yB zLky#4Oi=)f-~f^a@BWYb*K?YKNda{wgBz4-w!@XQ`g^qK^XN6ceLV*reb-KOT&%9tza-%t_`b*Me5+z zEU|FjNu~9E^u3#ef)h!zrYl}qICuxZ;UJ2mqo_0z7zC=F53MsWYFPy>z?34Ep+NJq zijd8a;Zww6dXPg?!UaZYxFHMul&SA&#}-hpU*Y>0#$lv=ea@_AM^H${W1HcEVA!f% z`ewXSXzDX{nI*pG1!IRsq`hSW$&CmiE(QH86UCX7WI6te{5l2AaV9M-*L)<}JSSWu z^ICEzF7vYr@oR^(L^6izH{MV_w(1qlZ~{Ps&U;tEGyz^YVDT9c^K?7CY#mQ!O(i^m z0g`JF=AS?iB5Np!1@bWLfHlApkfW&Kh>GdAooW^N5fByR9|}iAbRkUFwn4ZZ7<0Df z^}ui()yz%l1gU`bw8D;U#Gr#Y}2H<1LMudQ^9hG8__Mbcp97P8{{W!_B?&Z(Z zcv*8*u+v?kk*vC=5J^OZFOC>dof?{OxR8E<-hN!zM~+6Jq2;lesKSgR?Rl#Mg$JUv zOUsA5qq}TvuCxo@5sZJoe`@uTZ`kKmqv4R21>;}puWKiu{2UfCVbY4XTjqZz1-?rK zzxH2AjFNjBD1sBVrQj=u!b z#Rp3|AXMy>b+kG3n$Cg_#zV{fFA}ctS4c$1hcOj7e_Dust3tgGLbaowKqU#(8hrI? z#7#x*UnRz;`v!K~3RDbJD683gxI!b$nn2mGjF7d@a|W@S$QI)WS3y(Yly3x`A=9BH z0RD0SgC!-2p{s3h-~W!|NGEI5BYj=ZuQjetx3$hJCm;qTH09Utwn0yQ$^# zbZDJ?&K33a!00dC830nk@8inAfX(P|D*3Qtga{;V?tGr|`J1ukS|qU@cYG!CKp#WEu+JCYdszFImP(<}&v+ z#PJ2@qj`&)BELy-((W+wqBS{=YFr`0miCSg9ZeX>R~fiocRQ1{I*JFLj} zGV5AQn(nPs*1b4Mhz6JcfwQC#=LJW8a>!n=$}ra!*oUL=8fv0CBpkMzsS^CWCc4j? zHv6d4q*b-@hX|_8jEtxa+S%*0rX-&mj5qOagi)NJYk`@oL0|*rN1$?n>-kLI%ht&E zDT1j9TMKf-<^RNYf$-1_ttlvnA!WxIkL$}|y+`22ZxYM!AU7UQguE?MZ|md(q)!rP zg0Ecr)Hg?x68hn~-=%N-v`rP#aWpX!1i)dnjx!!M5oZ};eI5hB?>n_FS8YP?PAxe& zcP`p~+G>7}U_m9!gxkun^cXs&;b?+iP2%)xJiP(a@DZ z@r%iBd?b!T>p4Y8V!7Ssz_Pi5qBuzO1>2rcgbt0gbq&5?doQ6N+%q&#IjmDyFOIjK zHoR^b;PZargef@k=Te<{P=d;ipW+~vVoyr+^3g$;?i z9%%Ssw7uvR;@k;=VA>>n`;p52UxwFAa3eG?n`!E-98XzvYkBOVKOFLz8-|NlO~HGg z7w}ZjFe#sTu+QwElbFS^o(?wCo=*lHY1*?AG_^mtEn|2xliwfkzECPoqmbFI1d-s2 zpY~_=CM-sdsGAK=e)uHlqfL`&rRk7sUf?a5GP9?aY_wKsm2RHW0+$abL8t@5;=MKF zP6;_WQ3@`quaO)3Pu;6LTuun`Gsqyn=6QoK_o66p@9;XE*B` z1R0ZpVK@unOBBwevp_mgL9ecX=AP+D2p#2LO^R}92NC~lx^BsGK|X^VjU8l~)}az= z>mJec(S*}~k`OwRHfPjfIgla141v*n`oL+0#eUY){*Vm4%^XH`E}g(}k}wbJT#FtK zfV@`b3NwB)g-*No0p98jbKjeA0$}-M=B@I(1_a#oJMVef;@GwGC27tp^CT9wJ*2IR za|1_SFJav!#Lqv&nrAo5KtmqzREKME&Y0k@#*`-PE19({-qctfE>bqu$$A4yPsu^ME04@`55 zrlvy$<9?)#ugIiy+xUy|IHUs!n76cVR?g4DzsraX41+XmAap5|gtwQRl~d5wi9con z#Dc(FR%V-U#AOT#jDXM-Z==3}B_V(y{KM33Q{f610I4`a$696?(?ox87wf#I-RXx)ivRYq4Kbrq8H^{+iX-_gch{QCv4M4giU|~L^)i!95q!4u1{PJLvNbM zLJC9V7vUb&=um<5x6#1vjj@^LpUBrh;1;gdpkD%DwP@aSqElIrh;%=P zbk1lDk|~zVZAin(sL{;^y#u_0Xf^ zZv)6?d>HJV z@45F{(JC0L)rOC}mI>42K!R-U9H#M8r_eri+)=@R#S+R(_uJztCr&Q2*mMnlJZzLI zyfWj$2;aW{q&ziL_>>V0;N4}z^Q2{Ry$w?IkD29y%S=C<_RWRKZUBT0dY7l!*CP!@ z$r%CdF&1wG)u;v}*!70GSL%Y17s=UXUuzh05+RG0KMWP0=C_$Q-QMk0uQX1G z-1jh(dlgU(8-NIKSo6cODN>2|zWU&9GG-#zVnrVMalE(BHi{!O#-ws;vDwOCUWZ@G zUSDDS9L@R0xzPz5Yq?YgawuhrU)e`XDDWc_poEw`9e9K;{u$DTKA>%gCC4TlTw|3$ zki2E6>JtO;WH1nd z_x&oFd4@Wm*lt}XIB3wB7*hBSM)ebQ8q_xL!!^^n3ULFoo=ydZ=J^ z3-F1u9@}a6R6JXwB!|~s_V8zw3%#?)ngzZ)ns&v>w2Rq2+r{Z1-Vio)hapK~v|YI^ zp5kG@kR-6~u}mfSR=jua3hmlz^U|16ou3~xd0>itLktTwp9MfOdpm%@9rVTx9iDvI zOjaAtuL!tXY9)&S+qW8df;JLy=z7ya`Muo$?dq-Aiz@{JCMK8=ytnN{*r$8#3L{g!b@wUeJrc41oU|j@}R^#L4)tI%wlhP{yt94f}M5j&kRMURd*|Rj; ztC&6p$V*G+jXn(juFU7+B>1|ZHQmJOz&SiN(7diQ-_e*dI2Wq5hAfF|v|V@T z?BOtCo1ljri2)LUeD$FiaeI$ltc2u5)Z~5bwd&v##{in^H_?@~z}&MmC9d3h*CET)xBb~5Xn?%O z@9Odi(Y6mm3Tt>drnWBYmpl3sW19JTss-vwj@rUesDyEdSl|n2ckJ8HOuu&l1ByyF z>{+YYwh!N-54$H8F;ahuI>uD7Ef~)!XwcU(8!`XuSYk}Ft`BdZt6fx0v!#EUF6U_) zlkF^`00};&xrW{?)4ZKC@#|+)P5X`ceS#yBn6{FGmQ9}b+tv`4l!|bp{fKaoT;-Oz zdGz~od@f?vR8|g?d;uT*5$E&e$5G?H+~y_wivU+9+H-PXIdl{9ZTprPcT5j-WGBn0 z6f1$Jq%qRBCG%KX{#ga#@$5@JVXs#+NpL{SXp5WahV2i^?DXw-7Q0KXH+1Jjn9~pu zOEr(6K}`6z^s^^k)Ss`2p0IKf@Zkx+?On&{xE(*rZDn)dx$z*`Kq#O78pG)FOX}vC zYY)SSIbfn#|B%p4sUn??Nq{^!wO>;`pr0yyl&4AntPD2w=>0Wx^eS&CfiCDbFx7tbX28fg4@J1ysz(Sn_2!RUa6dffvRMw0_(2^0Pj$N zsc^HvPh-Jf_-+`q{Q&Y!4C;sehE(-sAroJPW{wF0?c9HVlXkT`2~mVHHDu>fG~&uO z!&3QYNCY(;i~|~^-jzYsJo=z+dPV#wkKp#82IA`+tu5*1H84?i4eEMId@d#)*E`w& z!3dbS52Wo+{{f4n%(1U|OOHdadeihG79cB_PT2BR(=fH;J6Y^4bgr+#xxUKxdHd>o z#&@}1Xw}P(DfFo?#OFA-UN&Mbjz+{@t#L+sK3E+zSmCd##mDy_gyBkA!Kvm8fxF^d z#_jXyMk%zXzMAJg0?iwhr*4am%DQ@!vuE@GW7`OXDQLCV4z?a4G5@@k*HC90T}r7$ z{P@Tl`mo%%1vZVTxIz}kgkamjrb9N9YGw*!3wn4>s*0Di;TNKGJZx-#Ufppt+%m*` z`^7)_;w-DI5uZ1}KEYOdM1S=Q`jB?FYjLt#GRw%eqoXj}4s4k>u8sqHXZ3uzfTk?M zE5@{Dwd{HdcdaL4qCyOpvf356yF@8=^x3_jJr9n5799UQH{Zn7a2q7lz$?E5S#$l7 zslpF%465LSj~{&%ll;yMTVW}B#JdNhPc}5TauqL9GyPbGVmAp zb_ePi7}u7F%@SY|)d14$6R<7!#Um))L_~377}+vLIqZ}C z$$UIIAzd~w^%3;*dgNK5T_WFaCyOC6Mn@1()z%JoYCsJxs7}g=MobrV+3Ae=>f0rj zXI^hOQp}czb-g7v5UOtgq7^ED&E1rwla$(dwQwPfU7y4|k zK%UT#A5DAM^uMoQK}z7KxtWSYP}E^zaS( z?v`BOG^5A+$*{cM)3MW4?_{Mz#7=Nr31oX!shJEe|fL9hS5pxR*b1h&d1 z$ybmhrI~l+b4!H5Nc?=AVaQ=SH)@m|a6R;18^c_+5)Nb~y!5JsjHwsl8k~=%t@7tl zTC!(=_fRg({GLHW;9i4x@;B+w$C)RK-sBAgp%!T3WYc8d_{6=t(im=u_Mc}CWfKX? zg>XEEmeZ=(x@}h|(eK3F)=x0oMB;n9K&-GZyG!$|K5VNBw?8thE^t5c{L^O0hi%TtbHKdA5l*Hg@LZGw$- zsiPzkL86KsY`&oQ7~pAwx^#iZNc#=M8oCM{LV(cnnkyrLvTMctp*MgrQkFO0FjDcG zcWR4<#UKp}en(4{^Tzu|oFPD5oJyolx-C_4w0x*hFICNMg3zGOhmhQBoaE~1h-x41D0eSl)rVe1-!*-`Fj#o z0+-5TClUDhOO??OY1w%!$#FOkd$W|Cc{-u*$NEO_XlxUZ_E;R)apf4W?wB3%{s{!@ ztZxKB$YlGK$DcWpATTDJ4#=nTDdIrAV)YiItdzy1y{4m(qj`{-X601J=ck{Oocy{5 z>%p-pVAK}~gz(d;nRS{cKRFF|WKF6*(jpSl!3*zD<^S7SbSD3@i~FB2J%yDax}^{9 zBnMH*iy*60sS^zs{NzH<-={efV`K$OLYGYFq;T&JfDEc@C60u;&Fo9?X=AMHAN6Ghnv1q~)W)5E>&h9G_FvF;XUX}S?wCQWD_}Te(d6YsjeO{uH>+kr>VtFU7as4e2M7DYtJD% z*1FI?D3?C^^fzB}8{R5$YZ2YU4x*M&_2N8b0wijmse(@~auig&K zxwG4xcYHf`NAPpx!K14Jo&bIA1+N~0)B=AVEP(7$iwqX#!TpDXwPvRN7f!V+eQUG>v6>PARBju|r}R41_mcm&a{D8(k6)vDG99Xxb&mFv2 zA~MK9IUqRx$s~Jx)29=IN|5Sri%xHYl41 zqVT)^D8`DaU|#}r91#9`dJt``TeBZ?Ja|VU}*Gx+GQ=9 zg^I3k*1`0$AR^-fiZ`(cbT`}RBob)mFo?_iQ->8Re3T!m$VANM(;^bMOGQpOwLlF4 zs;PQIq_u3CjlOp$-uOEkjjkKmKJ}{0ZS}6)@#f{hfme z)S?OB=>X!R7gN{(Pc1+8G{`nIWW%!~O7YzhsvZVGx6SkDfEX|VMl_76e3$MRxVA|{ zIU)3(v#BI~!wBQ4HwW&>63pcl3@?0%WIlnqbF^gV!};W!OR_b0+1F+=-IUyfOQiS> z+a6f*`zxn1{k7{rO)1Q4(o8uefds2|&M5*sNX|ETGb)pyPB095$h^AvE%k5W7YL$5 zAR2YIPf_L^Yn0%|8Y#!vx2D|9kw zl^`ACmj#|okg9PL5gC}w9ER{(;>uv&)JBg_f_d*IN_rguWD9ZtW%5akJc0@YS|q?( zO6CYc>(*u-3%k8iOD`KmGD9EN>PE#XAr5ksUI+-7|6btBj=)mD+EukX@Wq|D-(}tS z5+nzIanP+DcP|fkj)qVvN&nM0sK9Ev{tWNwPv7LUR2nHo1A$_%8&Ry<%ftm8|I0u_ zR%O_x?GOorIPuGOT_T0Z_tGts-aJ4sUpK z(4#u$#CJf}aA$tIn|qf|I4=)k?6T=wTA&lSp#p*t1B*1fZY`d|gm-V2^1#Ck?gA@fsxKIk{w}}ISaIX!C$GWcCDld!O zA_*YN%rwi~T@(Ysvzj}L)dBjk%yF7lcx4IUp&^Rq?eXSw#zFie4KsRK7Pp0$f_N@0 z#3h>p#qYz(m&XBSx)K_@a$6n7bJ}Dfq?TJEj`jkJ$kaE4$ja&fx4s015eHXeP1$`a z9=zwnb=d%-&x1&nC&asqZnJ$&m1`OxNh~VXiEj;nlQ!#)P`1h-0xp=xyBQ^kJJ^es zn&N(ld&Lo(AO{y6761vKZIB7jQ43i1-v0f0mzy8NqlZN*fp=e;R5e{k8!_FntEbT& zS!XL4$_wxzCnIlqMDpWy1*ZlA$TWLAs>}L@@g7*1As2x)lt1N(%{i^9XSV^=`$22z z{vCk%r-Q=pefsF*z-+rMwSjnP2|uIoU}EkW-0WT%a7IWKInJHi1Tj%F&0}aH6+zXM z|8iHoJgK-cqO?M3ND_oR)SeZg>p}tSDxUuKHX?;*bS>||ZbXOJ@$z846P?`Lt8oMF z7uaut!{_U-NUB3PISB(Lz<<_F=?zgOSnayeyEiPB(3+?MQCd}%R^B!(Mx+?N+~e>i zLRC)X=aTQNhP-KU*+kId@P~vtUbN!O>UW`Yj|K#0Yk3)`bfY2ajj?~F{3tl`0Da@< zy3VDP+0m$iz7YzgpNx8Gx2Tki%My;R|On_5yHDy3(1uD7{)c+v=7VBKiw(mBTV z69bf<2>>Nt1QcBudFRb9K8=7VJ8$vDX~XkGqcv7IdagPgXrucl_44S9MDGaJNot`B zZ^_cGtqOFodf3}G7zS`mC;X*%d^Y6^ZoAEbPftyYfgCetE-DjfdDNQ zJlcdsuk6wr96`O0BUDvfD;tkXBmK#I6IS56*=#^mf2M(PO1T8N0n*u&#q<^Q_< zBkPLA*5d8#YpkSz$|90VeGM@pt`70|P~8c}CJJx*(>j1cODOTmT3QluRd{l7*5E6v zKdwwU3wNf(INc;<{t{ONAbU(LMH8^f&m!QEo2G9Zj4D%v5BNP0v3r5>`^Q5M`{Cw) zX>x0Sa6H889yuuly~zV~Qle^uaDFGgRl5lY(S=Cm)o7~TE+ zwc-gTiU{$WO(h?Nd$Hb#_LtA?bRxpUGNl@%Y`P22tkzos7esEpVLv5XK2ueK2833{ z*2XMd{I~WIBq+X!J4I~g=q3Vs@gO9$b7|CfX7F|c zA?Nb~uX17aV!=>4!JNX*O1PB_V5;d^`eUNqm8#JIm1rz+`}=?F(jV+{zdYZ$DYf3a zhNA{od$ws$oxxP>z0eLsyRn9!F{_q@JrBW=?{Ha#!AEGRgG2GIvZQ5M`rjJhpEM3D ztkszB$*k(0TseAQgR)YlHzlhs;2~$RKms2Mo>puLT5FtOUg)^prBNyzWIBBj0%whE zGd3tbDA1Wh-)YiV#|V+EZ8^@d?z%<^P-fW3$L-lLJqU6Qy&t_pa7z!+iv|!^P+$a0 zdbVeU{qNa}`&TI|C3}~9!Cc{*1*_djWWQIDr13SXI?LT$Fh9Eic!@^JAR0e-V#l`Q zmLPmmsiv;I*D~F9X+RL0k1oIUv{hJ)PifTRFILB|kJTF^K2s81BmK1mx-yr{R9!3d zz}7h;(&`!ZXi(j%gv(1|>m{vq1R%oT7xekR`XBrbr3BRYSn@W6H!DC$6g9tvwF_(V z2$w~-$YsdD{~CQ0D~)4&30N*`Kej!4nY?b3ol&Q@+q{KTl2V&;x8-|UyOzCM*zS-i zx1~fUzm%oq`ShLj$tGp87_ZI#7}-195`Rw3u0_)&y^%NJ zDC^8XJZbC^5YnKLhTBGk{mv?14035o21LXI)R#S*w1_9QZkI-ge8aw2yY2m~Hkadh z6l!;HGkpO3y&weoJgP$PmiEuD2Y$8R;ji*48%A^@3Cgi)PrKeyKKNnw_ipIjcnrx3 zN*#7ISX6!TWXYPpbT@qb?9NTV|uYe-`gW9cOaCpvZl1SI-93H;7) zv3PfgDlgzbu0RD)dpw2qkjOrxa~{i_J}FGtt1Ms}q* z*}cQvMZ&neVez=*>yGsURoL_+<2Rr9V4spxh6W}alFC^#QtP9z3<`4vj-j`G?M$%> z5SG17gXznDqMEKtEnzGGYfW_Emw^4)rG_x~ zkCc^j}DfVOK5`QTcd%KkGFfm#VDIx^GM8hc4T`;(K3=mAOR+eLg2pHD;ocmuc&9 zTEHj4^mPL(Z_ELAwhzDy25e=X5ctPZhD$9~+?Edyrh-Vb)!grdscXH}iF|}@>y?kb znmVGAc}>mYzRX&T0tT*3vVYW9_O1IVa#$tF^A8BLh-@ZkJMjy~B1EM9D!q_gKZY`0 zW!l*&F_(Tcv9+>1^Te*de@G$QHJIou7(NyhKshVdixBMy-wXXju8k|ieI&!|56h?D zKk?o{@Af5Em*j|J^si+ReHk?hFQg_A{%>J3JL#LL}djq)M_9w z?PDGkBH}>#ksK6aNE7X8;8rj*|BK(~$HfkZC&qt>5vItoO^jj~A15mzrf(6(K31n*p# zQ3?nLPK4qHpwG2g1!`SsB7D#cXgC~&0vAW@wT6A`te3KK%VzD9QtEYAP?_!baV4kp zNtM(qhfb^euJyu$2Y$yjvi>X(A}+=zR6a+B7wI6$2%`1&{YHjo+oyzS=^d%uS+DeUXEx#SM+ji@6rFjOiWN zue-6)mL%0`U2;disb^Upl&)F!3&<+JXToJIHW5Lm>XSJ^7j1)*=uNm78o1Q}_qqv8 zRhaURQv}{HY;LAS~>3%#A1Jc+aP@*#9Zg_xVKJlH=EjvAW%BEh^pW|fNqpI zl)r(*0tyCt74fMre^j(IaM+;Kv@&Vsa0#Yv;a|NjxU&WUk=nZp z*5G%;u4XtM50rXAxX%yD4x~$T*}Hj1ZNsKRe8>zGi4OQZ#|hq8chWKN)#dm<4ZA)ilIVAR zQW{vU@sz=0u@>)UM+@go3U}Dh$*|lNE&%*g(EqknfpLVn^8Dt9y zzbt{l>M88D<(6L;S3N?&fadZ=6)u-$`uJ`Emj>P&9tCe1LZJ4^wc zJj@JQf(5bYt?L%)=3c#&j^v=fjVD{jOT?Z@F8!~25dBcScrCW>RFRRUXczKj+^K_z z??}nwyeW~O%o8&l3=>KD3XLR|k_$p57Pw*x;C0uWImXW(ccKKY%49c1=K^XJEHNEX za_n)5)yk2*Z@{qr+>*{>Z)cj*+uo@-I&{eaD(2#>Q3fd;j0dWgj#niP9Gf6^FnI0r z-I+6B^g9d)&LGCrE&e?zcd_Az%*@E1ZqLI`i3lj@E6_*Ee)(mrqKPE$x>E|4^^5d| z&Rp`11s5A9^5@iGqL?!{*{SK^$(CfM(1-nNUjjEFE3(t(gU%x{`dvs`4HfLY?Y>>P zvV5WHQc#9|8J&1$z4lvkEE&)U4Fl$dTcFm1!$jUnc4KH(ziai2#1N{&5P}xdsgSzJ z!l(x_mZg7S@TKMSZI!BSstAa={E@oEt}s!7^{a!_zyLZfb1LJnQjCC0Cl>06IK2pvha50G zm$fch9^_(;?|aR__X=WYaTsuokE`W!{){pOd6sSXQC*ZKq?bT2`jppUiS-#1 z2tL+8CFJG0AOPYPzzq7&{T!@;qBq0*p1>1!n|G8pd@V$HCn!S77hz@Btp%*g*i)!rh`|Q;&;LP18L_|3dcAx(~-<8RX5)R3hD z4nSD2B}!;B^Gc*c9Sj@g-V&9)Hjj^~C*Wd%JjI@PWDLrO^&0+RBLFXN-|f`|EL%X7 zck7zDWYW6t02hWMz0s3#dGXSp!m;i-BV4xEHTcUk64_vO=DjmXEMI5RB1T_Wx~*6P zGPUpia6>f+pR2$F9;dYQUZBu$?BaFHovY85hMuh>FPk9UKrl|sF2tTgOWHgKk^+Vx zNDy;u=4}<;9j}%i!pWW zY1z0DMAU2XdWLV$7W=bXEPuW0cU>5#nPq>;@w{ODLcjnhvAtOb14PmOhum%iY7clK zv3M^grCb#f_9AshvRlEN2ieJ%WP4Uz3NDCb(lDhEE=^qT!N?mbZqr>3z%?|UHMY;ggDow!>_cBSyts!ZVH^O`&Ds-6OJ|mQ#fJk_UfQy=Yv+0grK%`bhek&P3`Fi)G0dgI<#!Bs z!^h6k*TQq+8#KM_dwsTkZU*TJb?VL)fa0VQD&CBpdI6-#?hIT`zpf+k_O8&ZIw3hE zF1HYRXlds_#{(yLAcbmJ>RT!&1LL3TdaI{k`6OenkX(qf8Ovx9?XOi( zQQ__)+3?tlNWgRxSP_>~pMQNmKXV)?a8v=CH{Cm!YV<)=NEL!G9=7qR*8OK` zIP7_MgfvQ3ph{L60aY^0e=OE*EFBIc19X76d05&GiZOD~fIKuPu5py$N}kC74#a%t)aSQD>#~xHKvaWZ2w_8(s`v&Y-WbummRfc=2eNgdbhwD zJpWjRo8XNq!l{zrLvh&-eV}y z5)JWbp_&Qz(eG$T{vH|Cm~(`EyKy5BT(A0te|xJ&b^^j3qNpSLAdHej%DzW*Hui4* z@jl`xk-_g!NAKf}3W~})>y!lkv2vul5zXtbZ#}2`y`3F^2p#@BIlWg}d4SxCc)@T&Os`$N=Lw-eDdaKq&t> zMZMIpKf|w4$-s{nzWSx)d!K(>D)hcFH2F|H>IaJKhr_6rM$4s#tv#jnpW6wmkpRZ5 zu*(9Lfi3(f15< zubPRBubZCvx1WvnBSFx7m_WXER$c@U+pz$uBm2s5a{VT3eARbHA->6+{{Q-DnCwqZ z!^M1VB@F4%3?F&=v+t@>K+JxLY=~M*O%SbR(R!4)IOz%QUw^uBvi)DKD~_7vU$%d z^OgC(?j_i`cTBJvC#U$#HfH1hd=NHr9B`9X0>4?5iYDSO02vpVr0v}6fw3!3v9vHs zL*SFCopv>$tnB|T!ph&_VyHqv3=K#XSmNyeexd*C2Q|@n)vaOwy@qVIaHPqLJdHPI z@c*8VN~%=8`ybyn&Be;#?y>~KPq0SsLj8)pSyC~^Ih9VEf<-)DTbXFjcIWM;WVA10M-&K^mOpOAX|6^=~zr!*hi}Nayg{nV> zlNf5IOwfzY8EpqWAHl!Be~$mpdrz`ASkv4;$Fz+lu;pGIfKS9s@RV@oe>V3_x_hPkt-msB5o+_XXlRQfAI|j`r){o^WzrX*q z7MebV**b&R8-yz3KXzvMKQU3z_N*H0em()O)JBA;E`Tkv#r$p0-$e%mmiym(L@M(8 z``bwv-)9MUXVd<6r|@|+L|s2D^Iriva(gHeJ}xQ`2(eNJ6bYxvp_;?Lnyg5y#Q zb@+Hq2hNVw?U8bAVm;~`mZUS55qUU!}`Vc+lAKa9H+7rtd}%eua}y}>nWo){IK0h z?Elib2ULCK034iU&1wEWe2y4wci^rfaVTp=V@N<5R&)+*%P4Pr`N*xb-v0FodnI~- zwLe(b?N5tbV^9-FSzMSXn&}Rzd3NQ}E0z^50Dx-z+hU-;H9b z*TM!|1d928azw8fp2-W+Big$^x{SFGF(j3M6RbkZB5#XG$5Z8y9Y8`$o0$}lmC{zzkFIhYa#CM)I zdWyA=*91^0iYNnx$x}3Q>ruy+70`Sy9)7p!hYvSzBR94aIP1mWCCh93eVer>yB(5 zN78Sh-YGynjixaL0UEqTzgg(w`{2U_i-!EdjHVUt!3Hat+}|`GzFnPyjD^cLDOK>- zhjWH7s(0BU)Yolf{|F#5PaM0L;-~96C&6dYO_zE2K!go|QHY6z0TV$pTl6T@FR*ye zO#p3vjd{pH?j5!5SSt`WIfAv_@w~Ov%X|jcqQuAHQ-Hqu&G7xRCw(;x)ElVB!_}}y zY`=-F0hFf#_6aP`eldhTwann7;7lm);b?@~v~1*i<@y^Mt)L+mK=Wafz@b-_@1|i= zK@QNfHddxRoj(zq3V^h?>Fk$ERt3POA@I^Sa1_?1U(S+DE1n<0D*0~h6Yn^D>-(94 z&mISl=(PUzY}{*oii^#*>Go4avANixg%i#9TsE94%oFTv;gHsO@7<~9%gQW-C=D(U z^P;JXv_iJmHNvB*B%j*U3F2gGrhDA}W(R9~yW4&17Uh!B%@~JM698HX&^oTw-+aik zxk~d~-s!);c}xS-Tw$#UtPuHDU&hF`o*Ug`q3hQ6NO(e^p@b1cyB>r@j0*0#1crvu zQ(?{7-t*LHgdI_`zg4s$#Me+yh2T*}s_FdFGe1znT)+vQQH)F)f)I39ZT23K3YZv( zl>%q*OMvA0@BNX9m7`ew(uU82;auHHctfFBP6 zkpu}mH7U~p$o0xhBmR-=ePqPyEp?(+!&d18EDMp%ov99(JSIAJYCD~P5-9TR6M1iR z2OgA(G+3(2W^d^8%?FTnIR;|{ePZ^9kJ*&ZBG?LO)Vs}UD_-XL^Z)qz%77^IuJ08v z5Cs7dgAxe^X;c~%kOm1s8tD+}E)fgRp`@gwMPQJSl9U*Mkw&C@=b z%Kf0(JRTtmnhFPH?G9IhcDDB%*&x>%aHPCcREDT1lOzLE;QQEhoHNAJWt+0`*^PlW zuc@t!+oX{|o|i5c&=8f6Y8h3HH;`0*_S9A9a&I0nM)&E&^0l#Z;_M0(m z)_x~Yh^&Zf%e5`6h4W6P|GK{YI6bG7pS>dA2mxZwNantut##kggeh+Ao<^M}ATI_h zH4xPy@jaRH-A!wj!%d$<%5RiBmnrjEmf+7L{?jXi`QYSOfvW$1T(=QleyEp`$YKS@ z7o<5c=wrDC)laJe-|{=)z5zeF*!Z!UpU15Etsk+#H>Wq`C&n|Y5G^YFt3{bP9FdT= zXGLq~ZU*@^D*)8FRlG6LY=;*fa}r5yb79k<$r1Cqtc9&?LKyK`lSH)M8syu2gTGH3 z#4+mR6@jR~FrOhlqXEhg8{%Ua^DOfNHi@gXC8S*eGPzEKzGkT5|JMsybqzES9ez-~ z|8w@WGoHD|X*OExuad4vtyF~IF4y(S9z;3;! z4Kz$<_7G#+mZy<8a#+MOW7zTj9duoK5MM%-0Rq3-Qe4*`)N!0EU}*}0BI+McG(-$9 z5JKrADD@s02HsEzX`k9WL!XVRl?dgZIg2Q?PHahlU@F0ppzHU?^u?Zdb{sbqIS-*g zQET6k%#@;j`#>RBvxo>$r1*F4AX+Xm-TO~Z``$UzIuYq~0XOViODe^1_9{6x^1ISL zb$4~ZaGDSPnj~Y)ts`;eQuq#cia^D46wk?VBhC1hmosdnnEWOJLiz4;D#9eHDI$wF zrx^&2bWCZz{T9$mfo8jZKbdrAG@&+pV~_pETuXP~SK61qa(^RcN+PJ?(N5|F-*QI_oYkU%rYT!$vw*n|Q1^S}B7}(tXwi+IM(q&H_>AfGK14%I zz`Gj>5}pN%R(%gJjodZB)9OB|I4&=?Yq>OBjoK{&Z}^f|0Hn(cG&}tIZ1;qaqNb$Z zf$SxK(h{ITHvmPg7IxYNR_r0wa7k@=6^?ZZub!G0VwaJ&`b2rugG&G%(jYlwRM8NM z&&Cg#lGKxA{98b=OX`cfXg>eeGPrwT;0r#Fltt_P43$*8%w=R)@K*~g_-C*8L`)g_ z7S+xHrLn&!F7B(r3stcO_Y=G}G)u-q<(|IWK!g+2%Vlv-s3YN2V-JPA{qI#&)Hzui zIq_L~<-Hpn{=`o7)nH8OnIs$VKf9D@f-G_LUHrhHMKjIA{C;dISt=+3=~0xWs2LA; zx+_yaEf@XeGUKoFd`k}=pdlAhir;nDSB-!hil>u1=ny9YSkbm>IZ)7&nttS6zIOt- zWOq22e!KvA^MiPg1%5$Q;CxJ=L~Yn{3nELDLHJf1;slBOzNvlG%>4g(dH#HjzP*F* zpqY&OEf_$WQc0%Zj4CRVIT(h@0FNbZQ)3o)IIGTZ9DdYeDKj2UGvM&-;ll`W+Ei?t zmO!vM^<*f2s|@*&+a$Hr570|BK^DgF-x%?h9DF06_2Zv^i`;y9iHx~9r80NGu&Qf{ zjT-r|4h1|F#MCGUD!O2VE*Lk;NmZYON-Pi|)m>?_bBit98scw#DXi}_xx#Eb0g~>M z!gc>tHo(0UJBLhn^f!zC9xtx4_{tyq6+VUR%@Me!Ws7WOFbMq&&W&SA%NLL%l26Rd zlRw6nA41n6U($O2Ok9T;t0t&di+qt--vjh4-#k=HBQp-z7BTEn4(##<|6hoCl|Fph ztxVDX$SzYf={kJ*VX&mwr}vn;342Ts$aL!32I)6B=sH*O`cyuy-9Apu01Rf{=Kz2JoY&HMiKktbT(k(0L z-y{EBZ@)q4A-)451%_w>gJx)vbo8-4$c_=%7Iuw&>a3QW zf#OM5k(PJgt74ZHlL}ztCu9WI6cCm}vy6Tsc)GI$+$bH+Bqx7A1-@$tX0N9t`xnxY zK=`ckR9hF_IjG1#v>+v>!k5v{@A=G00ieW)X;%Ej29du}q#Nm2SA*{W`{%E~_OUzd z6BhMQAXzogZ$WI4rpEr?AY3TI6SxW5{{w|QBZoW4Xxk=tZ9mXk)siw-zz)z>u|hNp8lgfG7B+_!9RWP zS7e=Bx5Od^Yx$sR%xCP&Pw!$9*X-B+BG{g?k>r zVZOX50%hX0y7sBI=bSgmM zC3XmsKGlmiA5BdqzF;_4_IlVX!=M6Hr`I?OZz8ro9#rHK_nXmDoGz!n|?nLwp&fMigK!sid(6pdsdeH#*xB0Q9I0$iU0@dABsJ%pc z<*y2acC~HFCyIk267eN=%I@}p_l)*xKE!;7upe)>5S^N{0qep^s#`XL{Wk=4I4pNJPpwF&aU4n-2XuX`1yf)_ix+Wi#nh8j4aOxL@5+?9s2RiNU$9bqL zGcGLcCHKLy_#;Q(h7ys(H-duCl$N(oG=3hh}zft@QDca>E^UN|NMjN5J6-x62mMlw2r#@m&T-{rwlj)Rxha zOep(MVb1E)KcKOgm>%4D6!nB_HrgycSH2aMuKh0BNhqAUw|>cMHLtCsoICnk*$L-@ z8UkF(gfm2M+P<~Y`=AL^VQ$k+jceA(b8ETIYc96XwW)6nMc6L38m;2> za{IDGxE;BWX^y$UV>@cDYVPJgHw1fqBHZi}BJ!SYdI2#_?Q~kS)#F&lO!|h@M<1dh z>ps)yi6P-HG4ENMu$

zE75B?EZJY2s#m#_hVU!M(0ay@NTuIJ$yQ+I}y`B62@|P zc(8bQo&4t+2T#^AHUXNnv9#%a2w$*?CTz=sXyMY{bKX9}BICj0*U8U>v=%=w_i_>v zEzqGhlXL8_ON9nSu(6FB-oAI{%12rr1&QB+9!Yt#b&)Shh9%pNMC7NPa3`zdkUWc} zLY%;jk(-!C1H8A_$%Q7SOL$q~!Kw9|&;?{0?as{YeepV`X%h0KbT{8Y&TnK^e8>YDp!Z8d}}7}H=5&6^QJz& zjIQD_jd!Ko=hTY}G-He>>V`Jd*OM&R^W$|Vq)k65gAij8(GLd?5E5Qh`g=0liLJZH zn{IlOxLdS!-NY&>=9q0%*Lj^oTax{kJ_Y-r>%^2E9_+I5Xx~dJa!T!{jMCC8@Q@VD zsXk-6!C(bR=IAn0XU1+X`nc9e*~E#M@T>HUtDXxmQM$ZMH(}0?4(HE;*nMr&Gzty3 z#Pv6YIUHApz@lsXnED%!N#oh3EUnPa^3RGFJrg;)#;iwtiQ9U~XUi(iV%H;}kKA8$ zbnW{tui}fFAu0qgy9E6*y%OSrMpsVJiu60!>bY*aKQc|rD?3&becyES2Yo#Exa4a2im)2~M#R@jj7iJ6xp8htZ z|5wNK=YQTKK=6V#|K6_{4j7gqgXaJR$%g5#xRH?KR&TE2s}Erek6|}!;_eIP9~@7h z1Di%KZ-cV&~0lW1{B7t(VB~>Y^(G zLo=4wMGv-WK5BjbICi0JOs(FshS{rY(z@!&5w46YZ-SUFF=X6Ysd05e$A?F?{$k%( zx zkZ$ipIQ zVyEzD@-a%Ki7+WJg~ z?W|gR+(oz-z{oHbe8u??|NM0#=Urb~#?UrFn32&?*LB}(rZ~c10u||?D)|2#rX>bn zJ^|Ds4c(|YZF;Fu#Kt)uqM+m`aF_&Wj?~(*_xcjP2nNkb96Djj&kzj$dF-R6tS$oonOZnnz;rq+wX4uE+X5=9+AJLg)3oza zB0LWF-&bNE-A1ie<9o$p?b}h4jRc{u+(ypI)0?_I;^Hudwu_ql1hW*Coy6937R~IU zt9jhIjabTobb$)!5u#=*J9Zvz0yzLnV#hN0vLi(}LJFbX*Ovd}KO1_W7(L9%)%5xJ zPH$Q3j|K|7mOl9E_$_Y_2A_%cnXc1sLr!09){_Yq9@jgE=M-aZvbkQxr55x9&xCBU z*^Rw$CIlj^e_}=iA-9BV^{L>J;L=QHPHDV`LQbW)DFv;R`l_MQnkK3soI_s^`{+TE ziBw-fJQLkFyPm8l@-6LYz6+BGD{BvNA^~pv3(^+q)r3*AJOhxB1mR)1Q9WaL3rL(G zbTbVwbWpjiiIrpJenu^z<^Ol7Asib|Nn5$j;Z&k$KFj|1KsH z__FZ~KJvHZ=Urebj%M`&xAp|slF7**T><;}iPt6O?$n2S(3gzE^3#Mp=!wyu{{5m+ zZtw>Y>(95n?Nz_@Ii=O3+f_dcP#M!fs7ur5OMZxBVzIR^oO_2+sPyRvgN%_nWI`0H zs+a-CGU4LssQX({kCwEy(or$9e9BX%Pj&EyWLoWnIJD82#LuYSA>bK4Wom<7ii^Cp z2V+Vv)@Elyy(msXpf)|ve72pqgEyP_M=hQGmWCQ3g3j$o3w@dOYgcXIB zCBW<_Wq?(j&2AY1>=)0d4;iLem9_|}@o$i`y+}}O%0+#jb-DZ%V~Ip+Rk?fx$6ISb zKXddQ`+eQF@~E8Ip)32%5>p$mhg|8psSt!nVWpYKl+pg z_E{;)VVQUjPV8AYu}OcO*kxp#hpJV-+#kXCM^;JhV*>a~`#GU`Owp$cW5t#2tu&!E z`FabjXG=&09D>k~9v)1#9khktLRTKR%bA@B_#7P>A}Q{_ESV zmT&G0cU7DgNw02!ddH)u-t9!*I(AhmAz7^mq5%XNJ9nb#BHsomz%^kqU`_hc{K@#3 zi{THY!@ILt51EF)cioJ86NpHn|DI+3{i^^~I#pI&6k(r^QK`U9DjOaC0!nOcFxu`k z&ymD+NT99N-L98EX%2MOQ2SbS?$!Oyu@;ft>uWddHWl7KPd>#arE##VLb7V&+9V%3 z-Iz1BVEVPWc=HEA13JnG&qrsoYuSh=BPM$;~IewgB& zxz1Dap-FtVTK?2|*38*=glV?n$?IHUCiPy0Fyos+@DiKhfk6y+i6QD{Jwd%>StS~m z;$_X$?f99$gL!Z1*E1I5FUQ8i^|40RhcZkvW2&(4vBTTsmNzI#OkaDhA#$GVFJ;x{ z;}C*07(7?mO@BR5`$-EhWttx%N(BB7M~fuD$qo46YL|w-q-Wb%j$lSLp?1n?TF<5G zRtCGx@2`c>kp!hW_Vh8DPG9f8VDFk`q!U{-GEK>aJkN2KD_cpdwD=CZ=3@G9kM_M! z*fGS+Ht9IEnUsE)XF+in#KNWuNe)c^TAlk;xzkWq)>$5DXeL5n!hKU7a!=Z%crCh$J>aoZ$kkR3sPGD`z=4SR3ui4?*J3h;{nhtr~_O;HFi-TBKjL&FiOyZPHF1r zD@_ZgL~b>`Su(8`3$%co#S0=kz|nhlkzPD5mZOZul!8V{l7T!b`L@i+ zW~N$3sEImi*|EhSNhe7B_$(h|}x#EN)|J*I{3F%F`cYjPs=PWy3O0?Xy2u zygO~O3)F(n?Td+C;n1FBkHLkXw-oYUe z%N8(3JS|#U%i#F(bd&fHi>ad0CA?BbEzqZ!6jSlwMw5kLWBY{{qt>P+hEg>7j>VJV z&Uc!AP{xn!TSbtND3CcMRK*Glo!v)L!KW%&{9wCZuYj%6iY~ z9%XJ*&{%>-Fg*FXbqA>BM$d(t8z`p8Y!nQF1k2weXNjPM<|XkDW)%aJqFM=`YwVjUD(gAG)y=0rvQ_Wi-4UAmRFeg< z54F8OG=Zm$Y$Ecp?|5@7b{7O&6c#!&!jh@t+N90i%9wWa_aUYkx$emE%c-ZHufQX?i?MlxJK7Xwr}g)uLpIh*hH3#}>Q1J_un5bJQfj%_#?M zj4pjhM)ihW^%wRUY!+I@R{e3;HiA2DfVqdwbDd>hJM5b}+~*L}d+mp}6R8)VJ|zL8ij0|Pq2s~{u4@(*BQo^<4hEt z9SKs0#^ ziJNV?H4?W(U&AX*b&XT1?;X||kf<-O?*`G#XM8}~_V8P6!zatz(iCEJjP|O{u4w!6 zYXyO8=K)}iMZC7Gu7}bI0_<5sgs(QskFfEyedqW!vb(p6Yc#i48YNZ;SXj`LTM|S^q9jq7Lqe@tZng$vDcegrD;IS}QFBWsLo%SCW|6{kcSJ7A;Z zG6R`fzW2U+HxULVlvIMAG=)nTBK$auZ%xk7GxM)x_%~aIVbgf7Xp;3z+*QE zKewCkfG2tk7Z{F&lA^eslA2v(x`8gERoh%TD20<@Y=C~$3AuBu=ituy$%ghoXdHfI z2%3A`gorj@byh;7iEwZ-;czcKlSWJLCceW{r?w|CklU&y5Kncm_l(Bs{kvmQ=D|_| zH$}ZD883~^?g+=+QySRJvs7>0i84ajjdu^ZQ;nz;y zhWmAAO0pNerrR}6mjxZ@OprY=dWC+nSB6~>@(6C65+vDI&9VCPNob)oD?8&F(|pA5 z=<`jTV%@oOV=#@i`WdbxH9X9VE47I&l2xNwIX)uJ#+Tn^&@DalH28S~q%N$MqF<5C zctZ1BEAIpAviaL*iM{{{Sc?tw4@6$TH*2Gjjn#(G16^a=Sj_1%zl(PVc%V)^{9}?pyVh4oAESypp9!IBW`K(VNAM#Mb&yQzK zoxfkEfKPD;Bkv?H*poXwT#kqsrJ{iocq2#{9|?D7l2Cs@Y8D+f&iG~fA(g;`%_Tr` z;>Ge{-rUDc;XAExb6KQteSsE|ugCJi#c_%q&!d~FUiG2q;D#;Wogdulw-xW~@_g#l z*#exAw!pM0OH2X)9dDM`rK=x<2nC3>0r~Tz|6?gJ?}a_hE*#csnrMhgmLwt!mo+a)Is0*ObPSrtf$yLNl>o z_c_WTwbxlit$R~=%_Jz2(wLNJch-38HNBQ|#Zg==EZ`+3KdmP1vDkWHHc9S)o^CcmfN#KxIu= zyLx_hx&U|^aZJ-2CKP_cl+|8L%3plb|EIO_#S%)K#3BLEslI3Qye+%Y^A|BEsCV0z zb@xZN;ga!mpCDn>fNH4Rvf3=j@fZzPwLgTpi-Labg+R#&h`lbJFyDgbLFnjCEfLmj zy-On-KyAL6i*tTJP`Z^$F`W8U4=HG(I-u$er>2Ok3!c4wWnXCKL-J?@`L$qEaB|tL zO3HmQX*!n)qzKC0T-W|W6nWMxAf(yg`d~lfNj`JdLcfxd(*cJy`0-3+e?|9fw&TtE zZUoG=c%uKo2n@(V(bGe3T0Kief$LOqyP?eD%kO>u*R8w|_YAp}G$-D+3cNe~@TW)9 zSkvHOZyQ;B+2e8}Ujz}&+tB7i-u5&pA^@u^EBHjjb9gn9h`qQ<1XX0DSBmThoj3N` z?Hj7S{+$g*?YtJoQlkni2NFboCB{vL0mt}pOo#qBFehaDf@G!TpiqU~%0aEFsC{6+ zWFee~Y7&H$dL=-_BjSK!5K~P6Fm7iIc!#0-IlI_EM}wzNYs)I|8iYR7!Fy8%0A7c!pm~+>fP1V|T$NZSF%UWfR zX>d)*0Kr3P(DG1E+It!3oeR*sIH-J@JPMi1ZDRwS4fNg#dN2M(NfwO9H^)#;`A**C zT?Rn2j^N1icv{TY00jJ0-K?fC4)hq~>1JGMgKQP27xt+~c?0ekbO`PR|!Zrz6Wj2#%IuD1ZpmS6>0u~8r0DB ziy@57t0Z57Ei*^!e<}>CF6GVslJ$WIu%sCyIP%hN%77Z#7!L@+37SUW1eu@`6#^uM zLS=9~_*B)LHk8R3*3*LtaRzBA)2%_*JF4rerTW$T(_j5A?fii*Sxpc18y>DkIlkZy6z%TIh5F9 zoHd^krWgu}hZaRc^&?>uO_7L9z$PK^bFxn)SkR}WxxuB{wo}^&n818GU;>oCF@bM> z!AIM1d$*_L`@1Y>&;`spL)H+Sf|@kU^x<(OwyV_Z$D;gY)xwzO=RCK%Q3x*F=% z_wA7+?BN4^PJg*B8JKe!wu=PmBi?o&77((;9(Z=aXb zs0r9G!XwdbxyXv+^SI`Dly$$nouc+nAoH)89PBp5OJ9ko7t;eX<_mr$GX0h!$e)zZ z;=;)PjBIDayuM%*E%Q^yNQ{Y(A=N(FWTn4MdUV3by_{Nn*L;24sU!Rm z*1i*{@{59UA6cal%@H>&dio-?nk0#22)Ah}g$`iJ2J9OcBl+?$+WR;-=3Tzu9CLEI zgn4s^7_`F+TKi2_(cYR|%${`=t54M&O*7RBL9M~3F4K{RYmQ^ZrCMVR*Go7#7A4{M zDwsa?pfunO;d?@Ceq#Hs-pr7%WqW?aXe8|>4d4)U%ZEy56A6ffEO8$hg^y_Q% zpvl3JcT4jRxG*x#pxuz0!nQIEt6*1_nfx>x_CVoL&GW{LQ#s)0%*I@*4f+*r+yg?pSJr~9EGMq#eI(FatiE(<5w?y z)eX-<@>rYzJ56auIGQSCo~=EAe+&~ul68^1^kgvl{W2tFAq&^*2v+$%Mq?YZBiiV z*pRS}*ei5=PQ?LGd(`z+ybi50*WC2HF-BllgztoyuC*OTquYCQfA3R zXvPT=IQV9Kpd(_|VHABMzWcB)c8NmJQ8HES7P;fZ6tkD}eyO3Nt9B(!wN3IuvmYx9 zZ)-Gj-4-0t)WIMHSp7N_VD}tkp-4YuXnc&Me+;ul>6P+>)6#2#Vi{RcV=tF2MTJ>O z1^p4rFa-Why>9w5orwPo$Z?KO_2*qZ+D;mkEEkBhQ_Ca{`2q^<@uqduQkr^t#!O+j zX2HeRnwUsE_lEmHAi%+3$rncfy&PlV8Nzb_sqKuVIQPNwO4NPFQM=ym@24L48YuDz z22Ykwt2CJ&V+Jlo*O}jc;Y?J%)b$PyVklRZ(LDhj7lX+?kHIL`5maiXK54&41C0gd z|8obx?eb?(+h3CwKPe*Ux;X{So%3slDc4poYYP`Sf&49=`RFOs4Y%#rlu^m#L@y#& zTg6G-&@OP+o%LUb7dS8OE;zOl(h(3NQS z`IV4qVW=n1kM;pdxE)G7rbL3+F=GuJ57>}80-QAXHK-=(siPuER$wORi|ka7lvQ#L z9Dj76FZ|dJ&c?I*m*S?afeC5{&C#>R&=X#CgB9wC*TGOv&!f@GRF7OO&iDtsAV8o;u=UF zouEspI`!g}CIvPGaWEhXFr;%ct!KC(>|Wp{6X~T7>ni`-ZguYzo~iu_DITfU=STQ( zf1bm?F}G0iVwkT@rdktGSW|lGKUtu>;~-l)%af6!SZ__)mZ`zv1~q?v%v=yQaH|`5 z{**sCC2Oe8N}9ieihA;yNn1K@LW_VNab2B$hiNDK2}5e~vpjm;sp z6@*wF)455E{m8NchJ4BL`>^q%j^H#uIWf)GzagaOOoqec|3B6=f)@H{(=!b2cEI7+vLv`i zsSq4K2N9-PwJ6*C?v9rjg3&M+CzVE5-^H#}ZFuEW>{v>DLU~DDZsFTck# z?(NZge)@VqW((iiwNg4RiY}dD;mViv*JNvO5^nd0QXLr0)?nvfMRok;RAkFt6qQ9n zr&tq=|Nq0AR)>VBajkJ%|Lpr(rl2dZiFxwY@C^Ky*LmubC#jxcerM7kfMMWo^sZ1uE0hGe7`ezz1MjikSITTP}* zJ93;68?6o;de^|H23!a{`kf+h`^w7`!46L zwEau%_P2uOY-kliHa$N$tI}sq1n~qPJL(R*Wy#fuiziczApeN%G8_YP)qw) zK*ua0MFT&jx_$?Y;G?)VZ>zF;$F8;z;3Q;C6CiIl=1TeCP*#MIehUsoIB44?Icrq$ z$tJog_{04+t|HI`3aq^Aa!N`gsT@uRfem+PGPP+^mp4R=AB zOSRrgc@$7rxeS(zTcn;1qUx6 z?gV+HQAX<8wy&MkKHR%(Izh>yHz%Dl_5#)~=dT4GHazwRgwyM*pBDRyaL7g7L`}&qqqkBM<|hoW27^yO+e5AAwwEOZE`pN%O8;CxF))(Rw>KwA|X7T^9O# z>=KMn{7hQyZ<}vf{lMBK0AM0u>h`OX<3=w1AG}lbSvuoO0C>e+*@ni3x_jLwQS);{ zNd4^Xzk#Fwi2?pP+2|~MK}TfH_*K6>nz1w+UMbSsnyGo=K{J`Z^KsBtrB-+=#AgJ1e}F)k$F zS-xCB`)}h&-nHO2pil3z0flEyl)~!*Zhw?!dLAP%kJVy{V}_+GO6-M8T!)~?*AjR< zX%Yirxyh${<6(YxZ?N3{^I`+U>!=C3lkZ_B1rnw1YEwLEyU{1GZ4@u=EFvCEiA)vQ zNd$H%4%*PpLMV$i0aHUyVfDUsEd)Z+gVWF<>p|M$ojbb<%x$_b2bURM(NkjN{5{&z z<2jHaB~BG_CKQZ)xcU1PcO{p0F=z&nM>Yhs@*U)&^EDar*fjHML-Y?v4q+T(R*}$Z zpW*I@oC~|oRM^>YEu4BZ9ayS3rrBYpX2vIH63F{$1&~fxcG%fciV|3#?!Y(n|LKtz4()P4yV8OONpmZOc5 z5>1Y7@{mS#M*Li>ZOE6 z89;KVr_ljG!Cm=zQDEFyAe`5_4WWa`GUT15uGEm$Koqgv5d0;ZPwDg}?UPu(x zJGl$OCn`7I!Jw^z{)zDZZlLN)6$+qank47hj}mv_@$S=s7w(BC4%ro zlSvZn(&pP)1hJ5koD=u*C}(?9DGd4|sXIXCMWZ&Xe8{8581~14L=lm$K&<17Jhejb zO!u}^F3e;nK=+K6t^YGM^)Fk<)_w3hRBoJF|J%^>4LBL_)A8wVNrZyG8`W^`VB$cH z*=i4jS5#xy1w=*}IVa)Q&f{_>p92j1Jl3WjMrMp{fVqc6FJLFy5@_LYCdc(i*BILR zL_eoBXef^Hz;oQNAnJu87M%NJKtXZ64OXljC2*oaPRcS)}ue_ekn}05pY?c`j$2%OejKL-2!YegIb))5R%xsJyea*l8bu&OSZ1bffS&^0{XdGsP8bypAn z%~vR!Hz+D?wrQfsV{@4`Czmw63n~e*c2?j7?_c&vtX$2=a&%~7E8=TZ-2~9J#>?ox zg04hJA!~}6`u0CkTTdz?lI4)bpj#0afSiDQg17zaO>-k-L3(%&dZ6yr5TU-J=-ko3 zlc|xjaFs_dICPK|2=&uz=086rj#cR4%i0SQQx_nu;Ez<$+Yn1pkLeda(){#}(980V zyIT$!7^sp6Gc&8V>`g*$`C5kRT5zVz;-FpNa+8a)J)i-6u&vf^?G1V%)t7U`*B4$5 z1&fJKKtvV9BjiO>ei98*#i|`` z_r;V@=@t01Go_nQ21f^a8`t?9 zKRxv(#QO7YBM2OAR@!6>L~i%9K);@;v=;qM0-Pu$FOF+YpS*YRQy4niV{-w$;TnCO zvz^bJ9mvMas#vh@mh|hnO;BXQuTZOSuq$O`!jJOrR06sYrS?)si1nHi!7nFQ0wLV$ zq!t!Mr@xTyBOG`mM|h7@I|fHSvgYFyNf;P>c4gksdoKi+=&0(b5GyRCIr2F*Zav7Nqt5=Z1sOe4gXSc;A2 zVV;q@m2-4j&ib?g;g8#*f)GN@|DJJ8B*e7I4gO*Au;LA*3oI^b9u1YYmHbo%lUfPh zF<#rc*^MfE626&4Vz8$@?VjV8iB$6CKdDyf?e)?Q&Y7^2-db?DQdG37ENv@{_YjhB zwq1CxGH?t|Q}g5>R?r<>qsYvZKH5jQr}Nn|e3EN1oKpCJFDbxx zCQE#YXMo3$QSBPpzx_uvh)eK^1ld2%+R!uK)C`&-o+EpyVc${bj)tW_!4_&s5f6d}}b{>m{y;SW>yZEZ*{$etUlN zD=T@dt_g>j!8oxd<+D5Z=&WFZJ7mK$_tbA(y?E;KbsyhXSpiCtZ$A-GoG|!9`6f95 zA;B5fBQ^B4u3~2;)cVg*(8Lz!N|2JCFh3h|Ch66;chWcWu$5MW(cVRi*m4)#V6=zX zHZHbo`r`r`y<-{5+_5?vkU3UBa;a_wnshC`lyMI#n?7&M{8qsSS&>W2*Mw8!)tvw! zi!3wEXjC~;@cq`1kVba8dw8j-((FaHsC=-MP=0sXo4ddAPSnS{!RPqNgv-^j?eBm& zejKuYv+t3rvlzE`1FfIj(aEnKdxC=ws}m479u=Ire&eqGD9I9Y*BT4{=}jL+t(dm#d%7 z(+XVZi-Lf=65WQndy1aJS|h#BH=*;1=(>9Ymg(MMj7`4p~p;&GqP1#Vd3wIUEf^%>lboi}>8*?ki*aNYU+avyx29D<^ zCo!9{u(9uxisknDPsP2byy%arGZd9o`~8z5-4^yF;>k85%1=c`;T{N zDyL9Q0hs$*qpE5%(_QA(v4EuIrYwUiFml#1dP*NYRSAjLl(=N3;ODr^VzwyN9rB9_ zRqJS%^2~2OwXaA^0BoNeJy6Nnn(&G(I-l~$hPr7v?JebB4+rv;%;?e`+%oGET_4on zzw)04r}Zjc>QjDXHT0@zJLGA*MJ_H^n3b9Bys`1T;sRklI7CVnx_M<(*xxm$FjnJK z7ia?0KE$2(28k4rJ;(;sPHDf+7|1QoGshN^QIPuZm(F$>ME6}k6sTcnz4phz`D z?b5HjvU+~^jHm{Lkf>{B^3_!)bJ68?scVN*DeVp1Wz2EkVw2TB<^Y_Zc6;tp&rdLP zvy5DKdjya8x@~1CjR-5F?|m{lvG=!+?$miRACh1Kf4^m|Z}H$5GL!Z;4n|o6E=?j6 zMaQ;jeC43!10a&@LhGHX7CfNxHIc43bE@Gaw}r0RrH07Im$#32OW)y>dVf zX9y0VHJTGAYdr~zjpPrM$Qm0waYW3RmJawy=R{yfCgsOQccqGc;-W!!tCtui{dX_N zGt@6PE3EJ)2Yd4Oj^vi-UOAvX>_|9xtH*M1$riMEsS^v2b-dQQE|-*q@b3c+O0>Lq zS4khX=cyF~8!2})ABI<)=wgE``(?uU2~iR|^jIbS{2{u|$7Qrq#%HUBmfO0{Xly9P zUQfoARS3wt>BNQr&XKg3$FfZWJ<{>E=YX*)!)7!Da6g3D*XUB&lMNkyU!g@x;GY1U zoSAe*K})C8{a1$@kW-}>-6B5{yR|62ck9N1cBm&IO{jsv!Qz9xTOu2koveiSUl{kS zs(k4rg>wA+iTU?0*yw`y&WP#&o_vFL8iHVIZdGo7&&wfS)UV&IZV=xKX!hUSfw-RZ zJ>eEtrE7#U&z=)e;ufXqcIjbjm@F=KyxMGgSIv4+1GWq*E?zUeD18!{RitSoG*=F+ z%P6T9-_-5$Hi%QNZl;axqg_cj-JU$$O}kT88JEytUd zxoEjER>uZseN(&Yyd9tQ^lWIbLYo;|{Q1R@dXKzip4=YOm3kx1sfESq4kp1cq!;uzojq zc~Lh*W6vYznb;jFea4)M12$Q<#T=K3Gd`4LK4u+l!UN+EKTsbqrDhfq?S?HW{RCK; zsBX_=Pq-euPuxzr6f+~jwpGa^oKIp zMfqVOfIqj$Zas9|XV2t72k2Ug(+t}oUhLZ=E@^KztRqxWO1T-vH{Nc<$>Y(L6kn9* z?`CnrGQ)+@SxHrG+YZl)Qk_!uw3p9FX9zESyjxm4icF373mjdA(1JNX4~yS=?9QsG zpp;|-jI6TtY0RQvmmUlN;JMQ*v zG!7_!y;xEls>qmQnH@!*vz@8G|Iz#ubDX1!=D>8*ZtHr1q2du%EvCz{^ey%xL=E_- z9EJ`yiX0O5=gb0|bKq7yuQ*(Zti&?<<|SE0c5I?hGBnN6ax^ES^i48t<79_+SBkEF zF00f-)j(a)wigL~q8j9_p<3*GLmhjq;6BM_=I%Y&}<&(l2vVqpgfBoy%+b z$eqp=>j03YMrY9ZXWL&XD}qVl_Lg3XUZ3%CdCG6c2|$4oyLU)$Td`ah#z1*RBB`v7 zkU93A<71bQh04PN#YlgSuE;yOE4(Q>IxHn$a<(o6>dRtM(L~k}yEKuTit)Vnf2{nB zTj$X9Wvo2qn#ZD;?gbxHK6llNOU)h-lF3m@--%|Mkzzt}rtwiT-U`FkqI>$Tl-TKh zkp(Ygg*L34w?ywNq&$6Sw%B$EgR^%xr5(XPJAhw(;$KJE8s6uVxLS{|%X%t#7v+LZ zRj$S@jt?%QT+R+C&CKaZ_fzK3!fSZExlUu(gJpmaSL-}lK5pz39; zrOP%zR^qU{=#z(cUvl3@v|>xFzQq&ABkbM!a8A{8y~{>+|5xi^$XbWZYt+TwY*@8;K?7-;D7yT2? zH(!9-^crP?y2@724)Ar8hQWz~Ia|C5#i8nb8z<)reTI+@#m*_-SY~WN=VOP%!QQ0k zWGG{q1D9jRLFse*y_NO%e#DL66GTO=Gv8Im{+t3<#U2c@ChW(uMnCdAw<_`(di-VH zz9U_!UC>yNt8g7$K?VP zyvvq}-ke4rfwT8|IKaOXp(Yc8mT2wuWd~AxrDd51zzH_gafJ6Z*8z~xmhGJB)()b` zcheo$(qmP^abuCA$jtq-=Ir;6(C2Q_-twYZG*Z_!k8&QG(Ve{|k-JYN&Iyy&c?$5l*O*@+@v zRw^4It(;q!o&^;PHV#fF>H})lpS8;pnQ%Z(3YVv>5+d`*|EwN=_%e~bB3Yf8t1unW z6PH?C_SP@kB&9u|T=aqEw~UmFQW~ojLtw?U2f>2FLfoRK8Lr}9hDe>!=zd{wl6YS? zP$TjDo8X_f`SN9S{(Nf>G!vx-ftyp!!khgx*{*hgkr?kB05w@TdQd2QS%i;!P>^G* zZaZi*oYif8!)uDfd&$qj$EY&dLP>A=!1UqXG7_n^DC6K(xpm36nTG0uC{e<0|0H+9 z^DURu`SLxN%H09s87UV<)W~P(yylY(~_2y!DpqDU&f7?Uk&_c7x z7oqebpTKtQ&2GVqu6r)%eLqj5_lsbooSAwXFf%4p=dt{Jw~K4;oR3mlhlX%}&cb|) ztkU;bC8$I0WK)!aVSA!(f1XGtzVHIGFdXJ4=sC@17ZF41hoV;D?I%!!88TCzZ7oWW zjW$&>A1c#^9=<1Op#1bOGrRC)gX-&*#n0-p6u8b!KU+XV2v{9y5C-{3}JuxG=?P+P^*IcST&C))oysKn-v_6}{n z)_A>{Jet68l{}O-rn`dN8hr-oA99vy6v`k&OT@g4Qme`1lXYpES1x1 zvF(aM#oi>{!DD)F8ukCj-h0R8{D=ReBorYEDI=>S4Ya9l`dFnwDH@VpG&Dqn8NElteO*zpEJFi`~!`%BZo5yUOlyOvi9bzEYlIMjvl0mbO zAsskW_)cF}v9}sv24I=yAyqO0SkN+j`}AfFQ*KO?!jHyc5Bs!XdmI54nkfv)%M-Pg zhm%X@`rTBQp=lRVe!Y*bAxB67nlbMd_RJ-7^}sr{+;hI@a9 z;>uC$f6sd&hZ5+Uz|Wfz{M7uJ*#Cffi#ZM|${sA3g|{S(+6A{Z70LLE6Jzg>F*;05 z0TQpUD#p|yTop3qH>&2cTTJhb(_eB-u&ysp6i%2-&{WL6$GZ;%H5n&3UgXG%}O zxRDdmAz}IM>ipnKw-#;~dSms?DZlC?_7596R+0yG?mU2x;kvfieTb1|%qjsuqiz_X zRm1jFb=f2GIja52fc%jyq{*y#jN2`j9+nUg`dMzWK7{7!1EVVuQv$*RnLX3;VB5 zWlLFx@Kg-)F4G~FsNtp*V%e|n+1gSwE}mWa5^(`0uA<IQ3jz6PP$< z7AOne9S4ej3oYQUPWZm>mE~<0%+8oMA62}?ib<8;60t9*kat@)byCyPXDXxn;I9~5 zn-_~cXf2x9xG{yqw`YzFRV38;4{g`0mqb~9(eeU``#-fUhXsI7X4GW@vLSlIyD9Ud_B4U zFio(|FT0cw*PMmUin$I$dOW_J06BjxvS_ef$*&6eE&4gZj~B_R<1p9q#vzNF25e5R zl$D81`(e*Eq^>eK*n@r`tuaw>B_dK3`BXq>hq_-*xSZ=IAbJS=X?&>aI&kQi!%HSP z%TTQ4vqy7jIod9DzbZ2){jOZn%FioiY(baw(-#CgV}@=zGqmO`dSQ&nR}bXRkXaNyt9aHn*1$hR6G$y+ue^1=^@dGBDURdMinj_Kbj1Ug3= zLEAlV@-I3OT9GbC_deqbToC?xS-S(h*{zi-8^NY?#`nQ0ZAy=9RGRw0Xu;?!Uga^^x@#}KcIK5qDLq|p>3(im7@qDejii|pkh zzOYrD*gwzk)sUOjy?N z3r+`rv@$~$s_9KO>R@t35bF_|2n6-$?T5NpwL2FpksJK-!Nh;1izv}WeH%#+Wn9Ko zL|0E&-9<4A<~4je6}-);@xIi)k;rWL#jYv?xrSk+NJf7BbO;9I&h1>G(dXenI*ARe zmbf?<#aXo7+`mf* z2Xh=OhbE285Q}Nf1jISN3gUscKdV3cu~^8GABCe{&uO$^*%sBGYYe|sE&gP{BN4VH zN~AO&op58XE-_Z@43hU?Z>6!|v)2!M+9Q9FS#!FM^*wJg1t8_z(9$aFuEuWL?jf(W=Rt*=`BcYX1RG5`R;;f za0Bcl%hIzCJG*iKXsE()T=s8wzX^z^{C-+VZ`)Kj@Q&jeFD*LU7gv^#EXr)UD zD_xw1SET<*MjU1sV$IfbOiqgTN77qjxu1$MkASprls1#Sd~jLWAY}Ga+?z zZ$2Dt2M^yP+Sp|r5O`WjimAhqZp$Lry zFT{o`W;DdMs=4$Y`(Fsr?u zdqzD&5>_Dqh_0@2UO8YT!9Pe0?^Md5gpC=7yqDSe{WPXv$UNoeT5M^>JHr%*y;pn* zWfDU;sGmXM?TNX$a`Hu6uSp|O-KS>eioSJs(O47yER~<1c{2a90mpFD=G+UM z_Q&k?*pZ0AvV|0UlIcxeR|xz_J(m?z*=v=Y>m0L<=b(5G&3GfOPaO}Vq}3WjQ-WJ9 zZn`R&w^hk;BMlxO{rR(@#xGbZFz3-d@7T9Sk?R&9c_98xJZYcc zoQ+GDxCOJnnM_Gok_r88T?XIAAX*XZd1EJghmEy{liX7DE=*dBhI9L+v(MQ9R@)Vp zD)}&z!zL-(ZlqXF8iG}qpRc<0Y}P&R)Qf0z5)8-GN*nZoxIz7l4mny{aq zwl%W6;ghB5>;ty1cRY*<3)y<7@Svj0y$GY*l6zrX3`J+vkdyPjExn@jUB$jK2ZO@K z9?sL6C!rhBhbS|F8nL&02QYiC{C~iIA~yuP7m0KU0Hz4P0>I!>(be&L_>v!_L$HJi z6TV1oSrRqKuE^u}VAaM2$IY&q$~|k+}Ct#Bh`@7Cyjp1+?$cRT}QX1qCA0nhC|ID#@bGKHt z<-4jGvhxx}2Fn&D4YK=#o9~A9{SR)pP<7x2wG(v4%whL-W5q30B0+|?( zbIFM?O^MDL#EySBxz6EQiG99$9$YF5t1(#d=1{g0>i~nBxp-(r`no53O1}G&hjdOL zvqj%B{{8tf#DHp6Z1tabNRfKa1mvxZ_iehYkSp+HH!>}0^N`>gE|C!RmUIqPB2?s8 z5~kXgYW^-Q-Tj@c)nKBR{|v)|ofZj~ES(SJq}*+2*qTN2_{ft3**{AJ9U@E zRQljXX-zXU>4`m~)O#&*Vp0+Y3D4lqpr%xLJn-c%g%<+|xU~o)AY5k`F#1!q=6%+i zQBhhG|G_sBr?92k$uJHL_Q#rSIt<%D&3Tm9?tOroQqbsiy4w%*{#Vyu{%EsKBV_UW zi(?wnNzMCxv}1Qd1k|E}OOi$snqhp@C1)QOpk1DyQMXuH^v7;Q122VQG=)!Foyq7S z-%&;yaZz{1!v-2gx7iow zEZl&2U9Nq3rwTjTa(b-=OH>8E?8jaewQb;+ALAOQ9fr%7@}|wPG)2?-aLUgb)+5~J zimCwFS!-@h3ws4iHx0;9Lg+KUKvoAhg3M-`Xk+e3Vvc$br3n}+{PzQl+D-!oeQvcv ziGWr}MQFad3I()K{xl$D_c%+o9yDB5qz!BhhaF?fC~e%pFlR32sz~pbMHO=67(CVK zo+d>p<<+w-)0B3!kSUT3KR;GAI?p!jkYZ_5mSZY#Lw&2#P&_l9e6i&wKU3l02u#Fi zylAIG!{oS-9ve=mN*|pPzM|Mp<`-gjT9=Pr2|%%x-1eLI$C&$2Qh?3L|F?)6xd~J* z5edF?A-Ax;rc%__hFLp{ z>l%dEU>2L(H4mkkQ6B;AG6ZhCLqPbog%Sr9iA|3@-3B+dT4$0pp z{lOrmdDTk*-J}A5b!9`??GmLFr!1hb<|Smwsrv!4>|RL zc2)^h0ktTOO1P$rly0@$?EFzJC>E9a^-fe(Oi4sPecyuuD_md$+xM^ zPl?#UKy(^wIICx0jIyuFjYL51Fx{<96BTUMktkEKKcm0%=2s-oKfG$fF!R$Bo6g@( zoIW48P-X~7n41K$_hIPrPw>VY?dL(xs#8Z)xI)76IQU5d`bkB8o&rd$OP0oVBn})lK z=Tk8b!fI0IhEznW28h0gPM7iqrZ(MXu|59YKVBTa=SOoG2xoIqXmPY^?3r(5J=oxi zRT#_)c{l$d`^YSV>!6~_z`nSPq^jZ4btQ?rFFpH$j*LsT)?+c$@VN~%_-08VgY4bp zbRa=Cr~W;#fUAS(l5837H+PGcLdC@m6m&xw&i2TAgC-9;1PI58B`E5RN!ob(HeqzR zlC6*1kRle}tQkKH*SjWbunQvhdSApmx@vg46Og$R9{gtp|Gz0G-b!UAX7#xsBojFH zwMA%HD#a(s1dk5Sbth?xP?Aa+tO9lIVr0$8tj)9i=}rMUlbwarDMbF+(R0-oB$NbZ zum49tW$!KlSqnV@)qQ&@7N5o(JW)Un(&<)FP#uekf6IU)F@mB4K*rKLB3^C(W_Q5J$cY$fhb;IP2+;~I=&GRs4`%&q94e{-4WN`a6dBT6> zig(--9iGZW#7zjDl%nKj$0cI9term^rBRGM65QgpOEtBl%(pu|alRoG1y|OPfP!UB zVbTpp*{P(msLHY$S0SsVPvzJz`rqztGPH8_o_%#5sF8uuYhM0XR))Pg>~`D#exM;` z{HF&L3M1RR1UJdad$e<}Y;)gqXTzW|7>SA@;>G0GvEOsh6+CFz4p~j?IGjRFqgv~P*=#z5(!iE0KC`v44=}_*uIO){ zgy!|O&9%1uZ!!_W3&n=}d|3#V`YT+~Q z1706s{~4AmC?uO+Aj9?r?R0Vioq}(Vdp3#PB9NBch*d83Rd!|{rfYk$4iVF3-SECX z~E+hPBJe) zZ<`Gpa(6E07HGC+M7K_=zdoG9hh_LyDqdiJXsu6S96~Ze=H=lkN%nNlr-}&^?UPyT zIjZf+LE-q2gL2Gb_FD;~trUH?uKR0sf8PWN5qxGnpH&?D&*7Z^a?0m2u&Ox{Tc{N(9B~xl|+0ZgigSE>pCt=udDgw}6 zY|gyxoS7D6?-=RrVfR>gp-&(K!sh>cpml)Zhfmfu@mI20A5Qsypo|l!N?85TKDIKI zH;9C4%%Ye+v{)D^xhGN9pV%+Bm3$I#Tk>pmt?sh_j1Cpl3YJ@VvENA>mnV_ve|4gA zL?0YXme|y6-{gN4{W>_*h0<<)e1@q7>7{hN|a; z-CH??xt5Z4n`|!h;Wr16A68u@tHAz1+OFIh(Cmg=AK7u^~z&MSeFwXtOV0ZolGs(Dek7Sa&#Rm>ZB%hEqBnnK$luW66{WPy6hT z>$&e7XC@pQx5hxzlznCx)7I=I%tk>Csv7%-OJ#AYvcz|e*1&J)V zw+}SBHAyjD`HzUdxnUeX=LQBqrcBXN8@@2`KiVBR8#!iQ=Sx%kdeYP0YrE|*H^w~N zg`L93AYCdonH1gp_1SvAhE$Wkl(X3Os7$#w^`$flVFk*4HK|URaLuH^o7Gu@u?~9h#BA7L({if?9>sw zTX~qk1*;gK%@{r;e) z2==qQb$Z$4$xcH@2UTzFs8HADuJdoL*}*jA zk_#p_|LQt*R}?|F?xNQ8Tj&S9_%)d@P@aOHl6bz@;xk*tZ2;KqM$zUlrH?nFQc;xh zyv+>2K<|u*b83B*P4pINUfS!3YY`FHp3EFKk~xNYQm#Cf(?jG(8}I^>T!30vpC{5* zcrQGX_sQ4U|K*hN8OHaaTpwgw#tWI*L1oJ{{1TkdpTmJ_pg-_k-^1Cgvku6x`Mm&n?JG3=_1vJ!ULSD%82+ZBQocNdVk^7x(JXKjl?)+TzTqtp5 zZ&h4(49|&>zQfg<^_W)Nn65ORM^Gln=!4Qx^f!%!A{BBpAh!Xz(1(MW{g^*EfIj9# zNrayko7kw7RTNj8?fV*VBbi?5GH?tf$&(B&j3JB79 z^TyhZ?zxGc$eVsSf9)td_F2W@@sEak~w=f(kv|%xr)-|YIDYLXp#kOuWP|*qZVT-H+5^x zGEGiAop2+f2MIycVN^)@hA-%6?|(^5mj()*^xnqW^MX+>8+cFHV3>U`t#L1|)7mSqiw zXi&auH?!wIAP7}>kA%KYUuBnwz_T`}(d<1osXaAf$XJK|RI*1Tq*to-_A`S-nV2B1 zkmrNS!HrMY_ORq%fVdivdSv9(>QG%LwF3+gu(dC&ZIN~1cd0gjOoh5Hi%i^mt-SYM zTK^)bwGP$LR&gpHrJ4F5hPme@IaHQbuY4nJ+N8c31(Zd%F`G&8gpj->#hh6213`E} z96<5wLo!Oe?n!UDwEm^?=3Sr!QU$EjYmY%75amhgH6c;@F>ejrob5Y-+N=T|?*9?x zH*DUQxDBlu#BL}>Pj0M<3Wy(UZr_`U*#9MG1M~FMC9W`u8{#HFE9yAoFsd(GkT~1~ zDSA%xJ%fXKZ}QINS?I8TMlUfIlz;^nER$WzA`^GjwR~ReYv$NER0XF^`cvyx_m6nR zb$&Un=)?G&#`35IY~*x@Ea>24Vmj7CVknXPeoQZNdQa%5VL_FKENAHtRLy{zkDBIx zmLM3wkI!k&b|1usDcK!uS7pCF`9rVlwJ0T(J& zg|F!~_P$-OQ)R?JUWZH%o&X3F+@R#(F%rt^W1fM*m~f>EF=dQBRInozdfl%8-AdK zw@oTH)qk}53Yiqoe*Jyhf90>B^Vns~v=}v3Gj>@ZUqErjC-8m6E^53wmh4xTCd7&^0=2%e(&Y%0e`Fc^Abm1+_@+`WvXA{G+wT{eSOO z__~wNxZi9t$~iYdiidJPcq_NzL&?7VUx+eT6iGI8Y98jx;LZ3tx%d6597>Qz|L+AWJy*f5bjS^GEL zl{uR!WD~LFZhf08%{-4_gLz@FyiL!&GW=i#)rqAX?`?ncRuVJce(C*BcN?`;_}#}2 z?u%ITUV#OLX7_9`=uK_JKe2zxEA~VMWGuqdM(Eer%zrwF-Bug0ZU@yxN-tCKgT(wM*LF*$j>7Au&3BY~ZiuytnAM@eo%#y?-Ye@g z{)Jg4bxDQ9cFwNu?Q9ohgf%Dl2u|oFmSrfDr>h-tuNU)h$xsmOtkw-PSr_`u@@os) zH|7q{KkrCXK=$8nhYEf`cJD7m8IB$c!@r(W#4bruzWf1M2j20yx^16fJtnEVec>>- zp4-V021SO=S&@L0ey^RnV@X0hmpQAK;|#Jy{&8$$Fy?c5x=*~^%APA=D3e~y+m5T; z^I!d#->`-!cDDS?WRxOQ4Rv_YVOmYY5>y8D+$UmPlWWmPyG3#y#Q*<5m=OUPQsjZn ziF?|c;=LIMe*umk4%NQg{?2r)(r~@kurokk8Qoa)B6$pkVP4p24bg3ewn44`uPETw z<_X}y2wPI!6&C;wwA;63agKup+5q(gq{s0v}>NT^aQXX#RhN zvR27W6|K;P5;UsC4E>+2wLSljwCb4Fd*2FC*S4ni{JI>qG&E8B{kvJfM*KoWF)bh% z^fXq^;|w8<ApYjwP z$Q7}ryU2i?L|USMg2t$zsravw7mrCV^%H&SF10pa^8vsE`{X9$H%YTn@5+%(N--}; zMePn}Gvm#9BP(uLOA)!^+FmkZ<8AKt|99IIf&?bli%ZYgMW81@P}y(aFoKy(xu=ig zY@CrIG<W2~xvc*?>unjDNp=thJC})ef8M*d)mqWZ_JB^-_%$fe$r1 z4Knsa_=GY*RaNr4I2K4hbYmm^r638L7Z%I1iT>9uUx!$n8Jt|@ZRv%7^^q}l<6Z9+ z6^|i03Q6%XL$Fsl538%Rv6q@)h~d0e&aCFjcLT|tGY+k7m)8=c4-cNTdel?KFqLrZ z!H1URu&Y`djWy_TRoHMO(btc^lHrAj2AmUUxqFBO5&tL~*40Suy{x z{;oc&Z7ICNXLBdu&i*&zQjz{m}IJ7sf;guC>p( z8X_UTJmBYVu)4j{Mmu;QWBWZ&DPR;1SraiE~d&?qmK4puU}z2k7Z*^m>4FBL!eSkZe4VYrOZ9M^iA zQU@5Wc*q>%@oM6USWKB||9)-XW@ytPoJ;)B%P~JjRd%16FDv(28)F0U+vegE2?AKj z{h+8d?Oz@T%k=TO6+B0K8aO)rz-t`*g=`9;ZQ+icy>rG(#{-wZR&Tj(yZ8_!WA}0+09z3)#^ytWuTPaaA?wl;yvgA%g6P{=in{`EY{cw6vd343b9?vch#sw|>mM z5TozzByaBJ5wlw(^pPR=tmMY>D;QD{li>3c;gHr+hxG}gK~E(LcdnTIf$z};EIGN_ zT(0M7_hP=bJylBL6uz8sFy%0Tjb4>-WgeN9c^tQ2+iDOb^4X8K;ZG7wg^1-mPOf(d z_K23&%g*ab<~}HtPp**unJ;RrAE0rJyu%WFQI9T!tS2A$J9x&T7dFZMSIDz`V?3?m zy46zFW;{*yn*CHf&EA+I6~k^9?=-S4BhWrTcijG8BrtTRdag59KJ~)=2>3$L!;pd0 zevJs6>a-8m$p4z5O;NjOCl~J{a<7v=Zrk}EDIMgA&xcJbb_2X0;>Xt29{6GqIoTF7 zMMpiNbdTgyiTExz zH!)h`-GKkfXo6%zaOGNZ*&o77k5a>VI&G`+H&9m7FUc`32t}dIuSkP&2 zThTC8hf?cBYZCZ~UuW46a+r~Il5D&cN(F`!0C{dK!p@{ zdX)-iTiig`k>ZdAV2@S0il-Tpc$)1R{jlDH4zFo@$^IM6+1j9kb{A3Y66!Gl+L&kbXNWWE5ZL?T#iw46S~FS!-&v^!D4p*s zEr+qA+AWgZNnQdYnId?K2)7>{W`8Du@4U9p(|mmP_RIChBePY0PW#tO2w}r%cgt|x z+g{s60FHF$M>!APN^`uzH~hc{K4G1$_;KNd(s$gP_r^>0?{%OIZL&!ZC!olqm&B{| z!t-AIGS+0x!%Gu%UnD{D=Xl&j7wBj!l=;28L5(nBpS+^1C_oGrkV*%@lPL5edQCb3=$Rbt}xPDzr~ns%c zBeOi1C;9+x*1%Z_V(sfMgdzKQ@q`n-(quZ63cR}N_1@le`-XzXvu-oJ)X4n*X1L~i zJGah(n{~En)HvauCnrq#^7vVy%X&vf-a#XXei$g1b=;Cu)ZFy-bg-?RVE?v;rn@>fTmBSVXr&3b8(N0F8k= zX-~uOB&3&{GhhE@TDQOu|0>A)h)A|`+lGzp43vd6*sfid*D0baPu+bTZ$}-BX%8pu zHy-kq>z)0P?8mJiXQ^>eLvr;ms$30@ZATW+na32!Iy%{h>dH_^b~xGj=kqTWUpfqZ z66{w%jv=e8&RR-ii-W{3mbGuB35$BHEs@z_#WPsnInY;}qQUeQcyjaalVBzeO}Ekd zB|6Aqdl!WuT1iGMiT2iDCF20=>wnNqyUt0F<~GF+FQz*SSw^o=Kn35~*a;k*-=M(G z)--#e(59ULKRF1g1}G*LE&pu!TLdw^xzla3t{#{4idu82d&N^RB?NsEzG$+x;hiSm z?PT$|gw`^_!+=_mi+_kB2!Y{Bl`QtQOHM*7jp6^NVvtL2^RgDj+}qZsste*YClCIM zmxV()cDTH3EKRYrq=Q9I;Ki07mO~%U?#6MWK-0si!m;EL@{y@nM25D;!`U3lg&Yr%*@Yt1DF#Yii@1NWQAtcGIW@0vA69c29}RKMPXGXpgSLk1d^oT1X2I1 zf4fe*M;{SyYsf@TohXV)@JT-O+t_X`%g55|1W>wUL{JHuz5d?L_E z#5|^7FrLFRK}n_{u=^2WkH=k_Cxm$x%%$TS`T+>gP8O!D~kpkadiKu|hOHf5I?f~$3PD1b=B>Pxc-PWkw(TT7p< z$>@fY!X%d{lOe;s5om0pYWJm4>U4}Uw25AqXEO;5%XU~-1<&r$1O{{w{n~SqCabv~ z_CIep)BIHruYS?kU4Xh#`&IKHAPj>&C;w@P`YVw09&dav_=9^RA`GHN_0obIoI^w? z|KUbC;7j!|u!U$`CVPQayTA3$UX-_9oBB0A+c``7x2DXe9-HQ+=5Y~iUvHhdFU9ld zApTw@yGsq0>$ga55r-udZ#45Q6;iYrK9Zaj*?yA%8t0R5jAof}7$W3Xo%a`&0SY*# z5k=G!M3!GXHRVWX=8x(E@xRWVaTC0C%w&I5b}UXY_TvWR_1_u4IlW{eH{ijg&aMfM z%2Fx=5&CcvCwK4`_oC8d=d{Xa*IK#ouN(HJg%eXQ{d4y(k>AaO;Y#T(9)_p*L&XW>7S474J7+a54QM`(=$I`)mF7g26e`_T>#Wn6?jyC1&G+vFlY z#Sz{}9jNNC!iSKtc3{fozgxQ6pvF#EbzxNqzrZWvB_kfo#$juR#^%(vX+7%lHshDzmHquyz@?l`yg&g!!EAI1?^Cl3qS%qq^^y0Zr-Q*rnZ2q6@w+ z;<)_XRxZO1ee)Z(cf6?eK5nHT`r?rDH@mI``YQ3k9SwRM-5huOvv>2fRVPTszdQ5p zOX!VT>W%Pb{Mg5o7Q%)d)5WW_0%kRCK!}}fmv_pXf~&JE zMI@>&JmT);&>no=ez}4!gP_Gh%162mG>x(q+pBmIyYBia6DGO=L}>Qg<9?{deNP|| zeME}I|CW7NJ0pgKqMzTl_CV2t~}cK2!N4zP~2o6iUfY zQ7xSDg%|I*qRCH_Jk8g6Aq7c?-W$5Lu{&jib>#7dgcW-_jV_!K@!QMXal85xadkH8 zPGbn)!5a$0+)s}TqLiufIpt0aUzR8sz-eVE&C%$vBjB%)b1E2Y_@Eofx8P4s4~liU z^)S{#*ZqtgmX@Q*fkDR$8*5tr)V%uwq&6PJBlfa)GPsaKHtKps>sf|Sp<&kJj_3+b z_ZZ6Ibl+89-b1;=TWcr3oyzDm0!D5nrD`%ov~#I4f13pz++l_7I$Zl9+;yJAh@6rB zb8EY_5@RR5!ANN8R2`}$bDFWI)4V|r3%c!k@|I5BNN+^pKJMenm=m5IY={C}+>p8x zcLGNR(dY!h=^yYx)@4q@+ar|KX^y7m{+_9X=KX)Ne4)c)zQ!`&#YT&CB0JQgRLsNR zub2|jKfUxiKLLU~Dv)ou0l~E~Y2kzROvO8WT#wuSvYn9&rZ#Al*rR@(r37+SdwX}} z$+2UK{w)emj^9GM>f`7+Ds7Y(zKy@fON1Tmycv5D+IzdI8>bN){HLqY$~p054qKXQ zp6ED%H@f}rjgR1HKllz&O07RiqFXxO$~%KPj8BJJe^~>C^w`PeM0-8nBP#B*(%81X z^HNc5<*pt@B>znfwz7y7=w)kU$k*Olf3|+exJhjfI7P*Xug^bE@02krAn(_iowuv? z2dKM0r7b`Ao1x3*@#8~-M+_&Vp}!rPRwK4se5I8+du73R!AhK+gl_%e|7Rt{0l_l8 zEUGsc)@(pX`DYbr)^*v5xmm4FWbY*C{{gd&wOe}2rA_CT4Ro+o-hCE=7Oc z5`gC!wqbRo;1UOPQGyPhMQ|_{{@z`gYgQai_kLK?r7dVe6#XeXXCVit?JVit-`?3& z2M3DG<j-nX<310n^gewU;9Kn;xHgV zU6HvtUvV>il*VD;&#E9T5NTFn#%iN7flQWkulZ)@3o~nfys~yXtED7%dbRE2>1iV{ z;F^8JaNYg!HAL`lGo&{?IoCQmC0I9br-c)xIWS|!MvO74Ma7-$}5y~5d6IJv*!+qFr*XZ)%QRJD{nl2>v_f6XS(+Dz+xbN6D>%n&5? zENbek6jiKC5?2a~{?Q4Ho1?WQmLN|FN!S}xekQw~>{vR5Qf?XWP3?+joRX!buS(y= zwHFNk{gR6^RDHgQ@E@5pKJEFlcYi@My?VowR81J1_N$oCt=U!aa6h+3K47Zi8aS_o6{?vnV?{;)t+u6Yj zGjD@K_B=fG7z3|qa@-^Wvacl|Z#0SOCH^?4tHn*7{`WV8IZSk}dsbF{T*_*P!XSg; zDvfV!kPvhOQ%cq$;dEcICZ%Pc`7BDLrJ~DRXmgf#VYxR6juVqqRp@3(cof{4I_SVo zWLL*r@fMhQ>^FgtueOXSpq@>``8#3F`R{)1+)zCIxPVef^!=3MD!5nT6LQ-{e;xVU zBUpR7&_kGmD7qVYaGX^~rntT9_!dScpY^v8=xpdU$%e#_^7?cbFlW zIkU#ft_7g%Tm){_h|9)@22&p;`9XVHam|5^p|?Lw@at}{vIRxetIt!%DP;rI?_JGj z$_aj5i|HL^6a_Q7ah*?1nTENn-WC>96n99iU2-PjE>=(ftk|Wl1Pz4$ykQ}oDD#E$ zgg3*3Zkw^OZu@P!MPb#?%>rFX>KFJxf>q^kJmHZS0|@7u-;Yl|hoZ2lG{8pF#6Nqq z{^~3I2dmI>9+I;1EzRYlia%X9;?$KFrxz45AK*a7(nP;8W_~q$Kg=rwV^`nx2~n*F zh{e{MXEI?)@l8ZGgpRwKzv%r{cIC>HtWlbeDOO`Dg8AALhs*@%!F>0kLtVF2<_69) z?R?hVp?`1=Y=r}WXz<;Ui|e+B-Xxc=oP=D;yCYQi7y8{dFA z8)sgNXMH>>>dS=zN7~IwQQrYPK7BiGE+ylFpH2b_eB@)1%hx;7i_GSs8 z9K{&dC)heC5^(vt(lj^ZNVes+)-fG4o=)jKO)H}*Co7$OEI$FAxeQI&%S|S}!UWcB zArYbOYoa#0G^)NJfU?&XNQn=p5m{$;8tf!VOAyWhRy#@bORjQfjM@ZbKi z8JYck0~6+5oC_TknX4}D(s(~VKw{9mx7u4KNv~t*8XV82!CQNACAI2RfrM@5F8h3G zy+iB{4199Ia1u#}CgAc5lc!{XeQUS6*9L`{C_L zRjn_#i!4H2C??sND0EsZtZ~m96qQWhf8m(l*V1e+xsD>ccF*65q9t2iD|<3zAt?)H zD0XDNa)#me;se#^JMZBoip1$D@(?$JwdgLTV~Dp5Z{>tAA@p|G4nA`*9C zexeJ2kVLBtw_b5s`0LJvlB7MN&L)MIW^Y)$8u1ui)8+2sbuyE-82FBiNTPuxV~n<+ zPRKkoLPbEX*gg+?72Fmczqu-~M80@Ku>biXztg8ptX)PM#)45pAYU+2Ew-_-c#|-ys_}Tb&)hw(dgx`x461s$BXZQ*CY|FrE z44HtwOeJe9pSN{d2jHXwo1X}wrQ5L86AWh#7<@ii#}IR26TiY79AAD+PV*JXOHvmB zn>AHvMB;6(vR{CiA3_${Hrtk6XB%CLNXN&U${gFjxuGggv?@xiZB_ZEEYX9lx* zpFwu0NRaqN-Ij(_z78FYSt3tb@;r3ws6sB|F}-9)A(tGr&&hz&ZS*#i6@bE3m;J`nVmx+NF0?d!M{-g&^o z$PWYdxZh#m*Uu_=;rRfgBqwMozCV{Q0o^!m{l(y@S2T@w{eeoF+7OZWP5p@JEMKJK z33z<9nak)@n(s!C279dskC^zn@Zq`eJy1srd8B16QB%2xMrZ0q)s6FW#)LV~YQBe~ zK8!VafdQY9uqxaz1zN#j-*%Z6*)bnq6?h&iSJz1S{n=VAVr|kwWS!M8aE?Ysy)rwb z;1{Qq2kV#@32z;z+od=@>5Y$0o!L?6-f|Xc zXoP~t*744zK%VC{2tHO{0;scu`L1I|onQcbCb)>cnI~URPnwup~8GX%0AI++-%B0AL-BhCQXKe`hp`gm?!TLSnYdj zoIOEhkm$tS?@sHP?!Gqhp0|4FgVye7q0+|1O*$)o?#LkR5K@Eq1s6>&35gJQudK(w zXodJiJVSgOOLi~2qd)JsWWv@XOpgFd!c*V>>+Ph=UM7D|B+uR}iy@i0Nto<6VZF(` zs01CJa+>T~hVH6mUtT=#x1shUe1TzP+8n8|*%IriZzxtjy_^Q45J11b(S4oAWNDBn zTe9*H;Gq+igBD7$d1;b{dAL{#m;Ni2pmCUSmQSh6hYlY(pBfqi{h@ZU!2)(2B(E+B(U)v zt82bm$a6puv&b!YzOk+&9GyV%+sn-2P8>AWLl_@UiJXIQlpdVr7rwnP+!jXlG8^@& zs;5fho6nl6Y2k@C$>PVb4e5AOp2IX%Q7UZv8GE@`!Aqn+K;{p>y#3a)>c^i4+Lc#N z#00XvyU>1l_T&0~Z<{{&ji6$vW+fDG(bq;+oLCP@K1ARssQod`N;kf zNBiTUcUmX^xJ}quS?S+8<8G#X6mGQ}S{o!!SX1MM|EsL*GN#a+C7v=O-H(K4UoY~AT7*ey} z?L^AShwC;qr8GEf-{|Kf(cx_M3_$gaPzH7M0YDwKihbb?&{*TdkCeffsO88sxOIe2 z2~$57eyVx&0}+P9z)Y8?K|H}v$r@;N6$@S@o-TkM`+&TFj^Uc?dow5tDfaGG{z?r` z$1P5(;T};1`bPc_1X|pFVoIVB=lPYQ1;Ha*_LgFdg5hB|rSFY#w=!{HC08`WFzYCs6nGQF?I-a+gtRib;=05;OL~h@l@Y{+9!;g zNN3sp`mOqB+z)s{GJDitcy*RpwkwHn#M13o<trQR)WJ~3NHwQbu*$$gyonp8A zlstK2144{O1yLN#UAUOcD4&rRVG}ieaZI^voRG6{s0&o1!oO!S(L)1YTi>t%;6f!! zt!P~Ig?&?+5so;M{Cp>k4f0HWIcfh%t0IimTAQ%%@$?JZt1as1;)oO~7g&~C z%y@6Kw=E2{V})T=0IE=f|2!AWJfw)wr})6d zPYi62iZ5K0(y0B~Pj&ykT?jR4)HFeTjC*dn|M#)m5&VWfF+hk{3pe!GhuhDaF!#+( zAskUR(ZF;m%|w`BlLwtQd{sU+m~?&4eB&{P>}KzhK_Kh7q=Yh13$upIUakA*L0QSM-7 zaoW*H^xT@1uXF$Geo=fGZ4N{45V&LeO^SthFZ!|LT(`ue&Mqb_P~q_~{f^RPm{e3w zo{47L7=yeE#d#|wnRrQHbgJfV-pS#S&-OZaRAir2UFMuA`(jpDBx9Q-V+n2N#W4qg3h{6)Hx^nlt#5V2|s z>nXSWHC!S2@&qa*?+mQ=>a&46I*&x;7yicQx06Rx_9lNS#*LdEZq&^%dbR$@Tg+0@ zo`9Inp)|sl*E2P8r?F{YRViF*lZCtZvfBRqn$4FY5eIOY>Y*@Y7McUYd|#X$Tmr)y zt>l9bBlHwiV3sbe&a-)fL;ec?0)0c}o5u$~a2Fa&VX1r*>IhnSSFE5=YwDoA>tDVn z0Q)Rs*tzQB;nOBqeu^HGF%Fh5g(boSZ-*!pSnZI#Ca_kl zbp7fr7fg1F)u?}dm?D0{ZieZmEg2VQmqGhj*?IcpYpPfhd^Bb_p>ryv1Y(r1lFesz#Zhi{3?;Z^| znTtU9aG%m&8?!Zz>wWtaw4&L=u>B77=THVrM4s+9Ukw%kfcJ$aLW5di57xsVmnya*wr-WPBVOpYpxVv})4a9;*er72B0TKEUG?I9xN$!uW@MBKO{+8P@*3^M^UID=J z{rYtAj{Bz?^D15yJzu@SJ@?66ni*Zhvm)Bzu?{ec|Gr~Q- z0d{y^y3SIA+Xzp7xl`ktkcpYe#tHjFqMj`NwdbZl-BCZyS2QVojG0fshWCDm>f)-| z#X%ZA)&5xb2SF8neN<%S*I%}bT@!}@0|~h=$aT<2to$+xf8HOW6wLQ0??}zsU*!&Y zV+^J?(zI>Tbom8WvS}7K8P1PoVQLSf=tFgb>$`P@T9^{c!+&2V3zIO{>4PGwQ!hOOhXzyw3cUJLTX6^OF@4H5!KLKhONwT2lR`VUcuTd>mmJ z+@iLx--6@JI!eDp8Nvg|hMjS6GH)eiK_{j+Z0%G%39umYT?*CEH4%3g`8Yl;rK;=a z78>t<3+OfNW|~b|3nU5=^M^ihXEMfZ@jhK08u080rb_I-KRRQB!Uu|;9SMz!j=8V% z50Bh<_=;qLS^5|@Y}iq7O<>{C{>KDTy`Y^#H>|dQwkWOt;PJd#DoyVs2ins(wd$Hy?3Bc(ty>qy2P{Sr0j_XkI4xE63->V&*c|>CeOwcBOY}xUNupbnE!h~b1r5+=I*-JT-Wq0PzL;S|ryxuQh+lC`so9n7H zGxb$354KysRm3$Mg9~3&%T7?%5j(_}_#37_iEqyR2WoE*38mqtdTY7*=+el?tkfPVh)6dI5sWJn|_V+v6eGABcZ=URs&b=TeJ^L@Ur z-}C$9`K!}$&im}W_S$Q&z1Fp^Rp{?J?YA;PV;zv0W9fHy&<29T6_a8-p|CQNb|R{6~7eTT!pW^(ycH ziv(d8D1_=mgul<=%RR8nGtwYC6c;1mPZw;V$P2f?U`f_U1v`gN@y)jJY^`kB zi;&3CX2Z~4)ho=ah*y`7uw6e%nJB~LR;Q!* zaU8{9pJMA=%x{KhqwA~IZH7SXdC^#}qBqj!Q2OJMC9B&~GA`=wGsazFj^`sa?-?KY z667hAAE>y-Wp6{-QEG)ct|RM1b5g)b zv*6Md2>Mpi>ZO*G4sOuc^x3ccYXf}Hpb1mW#V?&s$tA4VN+RiA|hSB|E163Usw zhNnq7#-ok%=0Pb>U6uCEjcW6%&KsUqK6FDn)7Mfe@2pnk6Id6)R_$Z zC`Sr=MzLeA4zK~m=)7rZ;(PYihZ$b1>_Ft9wK)$ca^Pk8bVMpbJwb|b7+O}oi-Y}y-#?^OwK=yP%=U75EB zZB1F-DR)=FLlvdWJ_3nb(O8t#bLA08(MPfz8^875H$z{iL$br+8^vZ_LubkPiQji` zpO5yJs*CR9fwSKMGGJDCPgUkqfZx?Ycgtqe*8Qx@JPl5{6#`2<>Y6H=4Lfj$S9kC) z`wl(}iQP-^k+1&LRPN$FhqiINtgk#{7aAo|ih57`3#6#W;$?X;huwl7ugI%q#ZDi< zvfIqRI}ra7o5a@{vL!rQV=(cU2P)1kF_{iq!3*l+q`RkWJ}&gg+{Ujt|I9Uay04&c zI5r^a>xcwlN3?NW%o8*#Uik2G3X`6wS<#XMPuwDVgJw9_k@Mb}VT!nO*5c|gM=q+W zfewJEYkvFsTeB=H!WZi8Tbrw3vHj4Qfn+u?Ifj*Up8ut$R_w3pXS@PLF zR0m-8lTqok(FO@F@sWe4-eC}V?(a+d*ifU-+|0=F>4>{zmAtA7{`#uqkj6Y`fiz>! z4m8tH8x%q{fRrg^-Pm{+)G`96KX*X~U^&!(PV=?UXqkH)RQU#dwih2AKCNcgT}6{D ze?G^Nj<)8O!>%AQ5&0q+VQ|5LcL`3vDVEa-5`azAj@=JhISv%bxAGy^Z!`ho^kB`Fwi`p2gEyae0|#l@Ws3iT&K0Wxr9#U7qc6_% zVNM-HXmT&a!=;aXBM)B(u8?%={Ct%#eWkB^Vfk6m7sOyLl;DN%%mwT$I5kLp4S%-_K|9?{~lZ-BK!?P1X)#QiLH75Ri4 zUlnf7>&!IT6-H6V#S5q$#lhs{H%L*W#AHFiV>5K-Eym8>die|_;Fi1y`fd}%kUlJ5 z+Rx?MYmwJYafC!fdduF@#RNEcU3GqMUwX;~ z**vFk@>E=ui=pk^bvU9VSkN%F^On$UWFKt7V(M5jtV>iK| zcb69IwIWQ`5}h~VyY4DkhF_)ZeCm94+kBY#;UUP5XQk9Vc4{y-drRGavBTX)pRU5& zQ;c)1{jMT17>d@)5(E=V5yFhiiR}C)$I?$;)=HA<=e@pqU)f*t_`$NgTWxkN4$+PW zlv;$p=YapEKIGAt!QGb6)s79XClo7#UuU7|5sLS1@(>JIPfyKv33kekA&7CFd(eOrX3EihpbdD9!eC1S} z8G%H7gbg_*j1N@op!?z!QT(zh?0%G@LaLAnB|_2%@7#yiU0Hq9cdQ|Arj_d*vc6kh zE^J!6@XH##R#iLHD8-Y%MFHHbyAg}01qe==g~YGZR_es9wmue}kmCXEyI5grLKk4_ zA8clFK>??}u9l^E_M4>Tz8h4K@3B03Rcq&Z#daIxB(M7Yn;qL5#~m{;d6?8>0HD*l zf_j`1gIA2UTuOL=XXB0^p;NY8i*8Dz&DZa299l-*n~PDHen9HWZk$`kMD(5B!k-vZ z;!&1<<+8-$?Pj9N4q zi885qXlT@FAJ`(?d1@58SF;m+%Fvzx(q3D7vHr*sqPmV_1gRw+`wB_Wau?5K4 zfz6$KnH4=8P7u}ATTbI1H)n6&_>)rwD9L{(RVwIwJg24H8Ih+(`9PuRv<7^pyoZ`* zws#{}*%zR}#tl$SeL&Z&Ey3YAQZ=i;O$qKJ>(JIrz1=5%$}*4$Nfp%mBw3^j=@GBO z1>4>G6mu*9cv!u;lYR6A#DvU75F|!8s1|qQST4(DxG46(MuzgDFq}-~s;H~;A=Si( z!JyzQMtpYv%oieAm*!d;1lXg-k#*6N7q%@4bt7`3>|EUmF)KjRsiZWY^2ezAZoj!X zbPQbCRr`s%Lu=5saWH>@2#b0bBYDOH9nzuDRk+dff<|r2C=% zbv5HXBi7A+GjRcevt6ga2K$@|h%anqMX&3=R=u4Qjm*Oq9F1#QQ*FoEy=UG*sL7bKIebT)=H^*sLVNWiSiZ_MXU3%D4@szLH)(A{i88}NmzuT@8C6~aK6BzcL85Y^Q) zW*m-Pe!%S3Y3oRbYXrs5F6hF!WT67cFXC$Ta-RzjVtAGfn~#(+kPrd~OVwGbODcMd zz_rirjX=|*Artxf&7~~7N#YEDLCvMcF-aRVQxoRZHM0UnFjHlI5pSxi_0ww{nxY6X z;rH|osse6mOj$LiZT;5#Fyk$N+55V#&AdBf>(iSW+$I$@sh`kAMKvh#HVhb%`)_`@ z{iJE^SKsMJ;UPjC9MF+kyxPK(g&oCSRZ`c3R?&5d2m6CD;VjoQxQ_NJosO63(M z3NsGcAkcS=cxuOm)T>wJ2tM&bU~vLPe0_)cdp;fEc+aM-QLyNwNQOy1F(}l!dAHpR z*wkp)P`mziJCak2b&3P;YxA5-dcSdnMh$;j`R#~GY}<_8%-aL3Qw?VW#I2F=y7bc< zR(kX)>D#G!BZ$W>75-4F+!w)T7oe>X64xATTt<~sAEqgum)=Oo6RW3TnrTI zs)8NJoKDR)19ER=qwOsuuh*Z~819LK!m0(hf?*{U)2qy0^jnCgNRmg7QOu~&ci0(G zWG0Fc*`9VZFnvKongvcJ%>|)*4c8-PE;2h$|8Q2tGD2udir6AN;CrkY@!X@4<2S5$ z0QA_cucnrOy`=aO#B**{SmwDsEB5GOoOjS&^jNd4D+T^e6i16pGcgzJ;uwH8>dlw{ z%%%CaYF70~uc8rV2o(anc$v-lNDX40zq^3VEShl)%P?y0(4N|`kK~&U-#&{QsprWu zhUs2>nrHX6miwvQI2rVU@_7L<^~-e{Cxkpxjj340Gd%Im^6+bCst1)FfK%>!x3ien zak8k@iQ+RNGDhY}9zanmkQELv-8$hIe@W`J&2Iq2iain>kuR0zDL3WeI-TuhNL&tv z>h5{1dGZQS6AKR@J;YP`5$bsb@i5<6e)ZOLl{`kd`skn%2OLu!-(x%H9{|V9dKa*( zB@#LiA57;=NsL~3RD4ag^|kl)ei*H z0&ZR>m!XkL_s`t-BeNPuHr9Jn9(f8kaMhW$+V|9b;)7Ur`;mM%Oyv@GLCVq*JUv`x zKBKS~9`WY}bMs9Oi<^-=I@4g12h{?!mCcDY&MifFoBMlBtYbxVq`=y?nGSN=ww%^v zC10D&LN;ROj3XvXCe+2R4k}BGdxrdz?x3qv@2Aa_t}fC|CK^0SL{eo+Rsd0vQ(szD z`*0I=Ejfw8{gi34+zu?@Q=4WLs#!H&<{8`37#d)@!e!(zDom#2iMpey3!W4%BqXUA zgY)tTg^e~8eW1Gp0tYt4*I{*;Af(4d9U=p>FYlAW>0qU`>tSj9yNA|pmUD{aOHNFP zM|YOuG zPG(WpJ%#kEEFKJRX!3y_TWA4X(7iBovir>R@c>o`-K{9=E}H6fz3Or$n7b%vxm@Rg zIK$@-2i>EM8&B0wuTFNCDXEYrE}IS0>ehU(JK3>L8=&0;jAcYgg+!GZavHd8?{0oN zC+1b5Z%SQc{7N3NL`S+uzIL9H^5ou?x#^Ls*XmrkHvfq!{O+-`buqZdaXlz<*q?{I)|7>}4D^uGZOqo>6?JU0tgz#Go_H`9c%Na{ zcl%ZW6EtMFj>V!RcgJZUNUANUad_~0yL2lp&cJ&Jay`(xXtyBdW!;!BsS)t1Be>oq z>D{3GTD+v6m#S8{|Lb-|9Qk}p)+=_@oly&FPPEpYi5YNSW2O z8S!a)mZzH${C}7g0bAgGXkKz^IPu}k2WO_cT>c~f{1cV}G?5jnZVXYSRF7b8v9`&Tt$JKvY0@xzNZB8?ElT!Y3t2sb6mV5*NsDJ)@0LO6DpK3MmO-yVf0KjO&kUK&S-@m;B1cYL0LUa8%7wil=PhpsO~m z%g02@gsis4>2OhFk#S7^x`DegOS0s0Hcs$z>}c_Izo<3(_Pq%qcftrbDk5C@MY!K) z%B&Dk;C%JUt2|}oS^H+Z@{+gKT01{P*al#lu+KGXSKMx=P%UK?({--u2x=a8jx{9A z=qkPa^AE{aam#}#g;`AraRqH#%EflyGloM!qt#z8sD}jHk^?=75SE;m-(<`owBfZn zK=lIR1$@%xJY*qN9}~pd4&Lk%WV=<{vuA%Ra_qFgMfavsXVP>B{-FzM;|Tv5bx3}& zbeQsqCq9-Ro*pIs!O}BdzIKEcj-R6@)qjuX=rT|A* z3Ey^P2VtK@z9F?Vl2T&A?N_Q!6ni&Ul1nl)^VYh`3^Rkz#@_bdxD{Jc)-To>6EFd{ zVIn2OWbyWhf~jI2$S5g z1?^umQ$`SeG7@n5d0_&}t?K;d6U+pxveNN%rhLpGK(h?$DsC1WFnoP;)fQY}hhV$v zHZ0i?zQ3t@48ZAk2~~51(sQ>d+2povwq)>fS)hdjt6#lxD@m1~!6|T9+OneIf{iNr|yGgEsgVg?>F`wk$po6o_1GTn^QS8E}<>Ics`< zx%_#t_mk;5V}Y8>BV+~5@}C}Dd7y|eF|SQz>NbB)6V{vthOww~Yu3XVvw({vQB_@t zwa{KYWoARD~f>p%Dm-&q*+=F?kC2zKH+mY6^Z(F!z^O&160zrFVauK&F6ALHV1i5_Yz_E6Bq#r#^&v z9n)aN%8RpZ*-W-e@@_KzuE_%ia}ADSJc-FO{K7Q;l>Y9`<1^K7{a&`b3k)u= zE>~#(Ztsv=GOx+Okn1MtMfJ|zKj<>fpAt2fZi{~GX+N6M4!Lji$bHz zcPi-hGnM$Fmg3?e6Hy-TR%;Qy{2*gKzM4NnVuOG`#YKm1Fzr#GM5HbRGiPP7yw09e za|VrP!IJ6*3p)RPZz&K4_AVj zN21_O#SdbZNtox$e_7`)8m;?b0+`Nn*g9D$kNlt)UDQHdoG2GGP|_IZPqLj< zrNG&WX_XhHQ3GXvioK{Ro#Z#7o&iBmeK+<ig7 zO3z|R?gp;MBV6uBVNd$3hD4?0f|qsiDJ%?z#bL%koUP*4eVN;65k~m-^r+>$g>POO z@N^6NN9iZ{#AHy}Xee-^3+?qoml?zY>h7lHidM~dp`yhm49oDw*d2A#dmKL_dgJ?! zTLwQp*N95*M%db-c(=#sAVNw6?ZIwpwnOny`L&zQbZH`k8h@&sUiO2#oYgJogz?WE zBvujwTzQ=d6+o+Cb(YEtFC(jP9N(V*v2GoQ3*MZSTGQS5rPuFv9UcoHk$R8&(|UX= z(ocOm2(frbFl#xf&C=ITtoo!xu{)@9I0DR0lCtwiEZ|UOQFzT<}E}I2H{ucbs$2rzx9?%th;eA!fuBW;Z z(}+R8>>h&YIJ)#nPW*)rpBaMR-n?lvAvTyJvEC+$;Rug%nxKX;fIH_NrxZdW0exNx zbV;`la}jBmiuI80u^(hy#vA#wq{DlxG6{&`hR3d}h?;|s)#I9zVqN_RH~Rt(YU1F5 zkVMpF<%P_p_&tF!wvC=(Js+ zf_||EzxY^N-Yt%dz;fwoHl--@BL2VN2S1W3Hqk2k_*TS$x^Eja^+3!b@LU*6T3@MC zO3-=AoAA+(Ex51D%}l}L-Ije}*E21rB^+<`pvQMHqHcmS_RZ9-Y(~^8yA?c>dp90SB0OXb0TXBnTKI(V3$-~t^HsypZ+XuB`F!J~QfV7C z%O1mp-*Lh>1?6@Mo*?@I{&bic! zN%8;00GT}f3HV)`Cj0XI6+Pfux1bStX$(%p*KiqE*+$IkSs^>Bn3ny8BkU_3D3^P4 ze|{vNq2X)eSYp!oBB%-)pcFbb6%B?6cpyt)s5P>PR)r;+l~kmBc9BWXk<5DE1j-R` zC2zCKxw~h6J#KG~ z12)LFm&yYnbRrWh?WeB590%DQi?i;wa}fFzyrDHieA&+yh!Ydz=UMcF#FBG74fOyc z^*P!8$~sekq(nHUMZP~&`!#d@MPD7XUD|@%wq%nTTo-C+@aTT_j|j<6-Cy59l<>m;*sq~_JZiV& zKKLq?f1>ZV6iETXJ%FFCd$DvG=Tl7R>7)Z>$>=^d30Z@nJzC`7om(VH8~}p&wzz)YL_Q)r-_}nB#+QAV35sB?MqI3A8!m>} z2FO)iB8Z*b>)OO!y^&NZ$*+n_$K&>heV3{h|cc)h8+jD(|x1Sk%X$pj)91sWhd^Dl2 ztkym8Iu}2H`4QqfCwg}P`*3@M&AF{psLzj8CIhq(UdLCiuC^}zbfU~$J(%2*ZIJkc z8+#^j*yC$o*Od`tk~;2&YQcWlTw%oZy)ILg<+jtJ#fjnLhy5RRp8|M?#AQY2uw}(Q zoKTGW_`r+SC~EyUfSkaksO4RboV4snLt7ka&{Vg>@}ljZR}i*crm7_dps;!PPI;QD zM99Yj`)!Mj|?C2!7I0 z@8Nswz-JytWM4-~x zEXQWtenxq?lQT@IbwQ6%esY*HsQr(gjuMGn2;x&YwTPdSf*3a2E!jM8%5;Hfe_Gw6 z2pPnCa=)XxheVq7mntli@ypxl6zEikEL1PuRKNj`rI+wCt_(NB)6Fh>?CVmm|KDN% zjl(|Myl^zacWSfeBS~NZtj&2cm)%<1t$Pq?TgeEaf-Wg_rrWdbK> zU(ccZVLPPVI4Y_ilYo)~+^yM^I{GVNrN;H$+_Q9ikK-cD-cmtUwhEQ2j|Jxbm1SCw zQ{JUH&o=3Q|5jCuv|*(fo%-Y;C(oGA(y5+{8P@$3Zpo*fzS+#h11l4NsVExa3|ezx>WHGo9?MR)iGp)ixj0L3dN|q6mg;u=1aJndIuMuN;|Gce z+94<(4@l{1m!pG+AL}=CuFty%zjWQO8cyJ?#dR^Snzy2yWXr4T%RWTK^!5$(VOWy| znZ4$=ZM+34r;oW$(Y#P_6)BJ64Waus{NDF+7zd2UvD3W6UwFyO68MU^FJkAW2=}%D z*qG|?=>*sxYgeLoxd5rMghTxY&m8uux96P+L$_Aq!(3(=w8+z&IS60Vc~5|_GYjfh z29P&cTcH{Ih^EMR1QC^|YZ&%O{t8EsX6R7uh@FzW)%ecVVz2sxKZldVTm#}jcr%s? zb?<$r2(a&(i^ocW2KFARIIPjytUK$(sLNwy*)??Nm`MkP#b+(?+It8#`t<}G8Phl_ zbArJHQoDFXRMPQN7p87{{ZCz9<wS*AXlr4><;^7`$hO<2S=U|f#!AY@!wndeI2M#gWz6n2W6T%!olf667J7$=RC=wmlnrqMgw07_< zH@~OK0qnuA=VzZ}obEJVa5U+s<%MNf8+T`w*i1yA$ygK{a! zSO0Vt{#LobY%(Pta1Zr89Ty#2E*8s$?p^o$y|!m6-G=S}JR&La!ehn+>WhLx?fp}I zh$*VpSu@%ob?>hMjlBL5JKqR6%TQVJMu!4`ug~8zeBmkg?WEML0|~ul!?hOos7ss% z7a2>jx%h`Hkp$hkqy+)5s6KON$MlYp>G6cx_bFJ=~%ZbH4T%UzHXPe~(Mi7VuQa=r&Sd;ka1C(y4R0L)Y(BS z4j~|=CkM?3SwI3<1~&b6Qa=s9*RS*r53 zM#19A$=9iS!WYzm8XjK|v&u&WZ!T|PH_T*bpopBf*Mjw$akqmo3&F5VugCK-A$>w1 zFDlhp;Y0WSR+pYG-We3@opO>Lr1H-=4LN+Vz&^$pL2a97ah?fl(?!>h{%6uA5pj z5qcxoV3XC8hGdVoXLEGF_z}8erxovigEEIX9-G(w2`vh)zJ4OfMmPTHNk0NUeK|J4 zN7*L7XWawD?EEGjR_#6^T?tLSPc|E8=)Y12WDu9Vp}oqbGMG>Go5pZdgb>ak6#z^=K(X^G0F`l zS|_{OvFy_#2!Q=y9ndil`IvO*d`cby9{Kx>@?U144*gT}y%qi=qJi{2wazX=eU|Gf zyqI|-i|a2BVQ;ve2-Gng?jxMh$(M^yME5V@oX6$s{uc$&9g|7H zMw}?S+J!xqpN063h4j&Y70}u%t)x%Th-=L{07o*5`)eRMUkwzm)iuS|8;XO>9U3k_ zw?~NZA6S4tZXMR|Go_#jfmva~OcQ$17)HU{R$-6PX?tt-7mYd%4=*Oy5(y;sarQHx z{U3J}QyeDCQ`vSsN3V!;mUt}~9On@i@|0W@221Yz5%&DYynnCK?hu}Oyb(V~I9_DM zo2Wp8zfP;Y^zwlCfu^V;>FLc0ifswCrm^2l|G!%~rUo}vS`eM;R8!rEotp>($66hP zkk~Fg(-O;R$*qS|Hp4Y;$eee4gb1#$cB~!f$X8f@;Q=e~@Q?iS@5RE&)yw!YtFaFY zHjkdfM;47oO}QV8Bkb$8g$^$asZ^&$kU{a!f8EVK<1oBq?+Y$qVU|GFJj_q39C8$? zh{9pnxX`ESK=LFGA zRM4;D7vqbKTQ{dbh}yf~%Rob}g+MId`NYvmE+= z>p>kmM8F8f)5Ghv*sbLMY`6267|>9Q1Jn|dADXyyuFxcYJNS*sYV7Q11|?IL)b9Gb zW55a24&@2LkQ4UX+@NZvi(%}2q5S}3#WZ?ddjGJGyC*Cri^WVE5CT|-7Ns~^`AvSo zNQv#jCn7)Hm`6hWRYJdcClTRG!RHgR7U>6KzUqQ2*q@mI1slPAzgMfpe|_3-@?tC_ zT7o5Z+OH7!cV+03#8e38?%Ks;IB!AbYvnAdZ}@~^%_u|pUQ2z#N*6jd**ebgLN)$> zx7)iV{d@1}f7NL~N_0*b2QvZ_2-|H{@l)>Pw~EV}d7Ke6b$MmgF3wg+zI?^=hZwQ{ zqW4s8N@ApI)RJzID(UxM`YqEaf_+*5H1QRu?OE9jKFWvbG$;@0 z0jxmAeIbnj83pGU6EVB9Jc~#CO)O$d@C}~hA=_b)2)a^HC#gU#SK`3i3pqGTD9i7- zTxy~+{n+MvImlT5<;2K&EFM#1x8lFD35*}WSUz_2LX7%Ydfd7u+_kU7 zsPXgdr@@>4w$1qHx~~2~nFsUVgS~cN4J{UbfnZrd5mMSm_&EOUVHsX{m^G^;vj^Vx zBiGh_p#2i4`I8hmxlRFdWw``@3|!AVzDjS!tpx;9iSgX+WE-C<@+3Ny>`X7hQ_*i;hO zX)mCz0c1$rGv#kzl!yl!{#zDcS zC*1ICrFAbJ;$rY|rm=l@TJRNyyL65~f5MUco2_8(#5&#^5x$(8kBuLjK#hGTCI@!I zOMQl5f@U7)M*hv=?*5q!912PevdVp#<#_ZhH%qpYs7#=-7jYN1?9 zD6g@udn#rW`c62X)eVgJql~(3?vGUDT7+d~ubldTKZu--%LR*C?fTBI>q`q9+H)Nj zMt?7;nf6o`Y@chr(=pERom!R13LGeQF<0>=r!Ld#I zV}FlnCx!o)S^OV}C1HEpEt$e+6oduRmhjv4?*r-?>nTDYh82RRKolC|&{f4Xk~0Ps zy~iD!k2_A*RG>;!@;851;PD;REL%7{@zk#J55&R&E|1&|9 zMWsJs*ZB-!qU25PK=~RyDNP47A{z;ct+4+p%+Rd~JphUS47vWNDR(_hnHDuTT3ABn zJZ!A;foX80<7zpIP+lA_{otMIG#hlrJ%X$M-^E}ynbHlg z`@O>fuoJ^}4aZs7siiZyw)be!A6@`V3pnQJM;EyHj0mR^lh_?hp?|cIsPVYTA zaxC80#bC;7uD#k6?e|~K07~0cNc6i9Iy*uyC<14sZs}jlC}iyk-^X}$p*{J*L zKS6k@TgLuJ!C!{A?v0MR#eE9K-){UL1l#N3>+rc*lWJnfA_75TC^JWsU*sspKKXnB zp@`?h)V}F^+MItS5-$>UeNB5hIsId3-KWe_N~_daoTplgX*f4FD~}$}=Y-{(cv!g` zB%Gd6OrOOI1HRDLY((ruxWYR68xHx>K zU*a3$j+T_qFor3nLU=X2OYYG0Y9Y~GljgemmVBu!`7kO4C?!wuFu)E)>oFn=jQZ4>4iL zW25-Bp&Y*yJBVN-q_6ZSpkU0ORIB%&f^j?0N^g8>088 zy~vhdPf3Xk!VQLjP86LG35cVn8VXGhU%EGelBY<7`UC$j9#-N5tsNRQ9p#A6#y<1| zqIUU|6_2?T&>hG9vzBy^yJx5O+DQKE-XnI;Mg0gK#{3rT|2N6{Q^HkX`AI&10%i$o zoTQW6mWi?r*#qD1drMEZOBhwiSBSjY|E(@dB+9?zK@hod+E*|wDI5TyYuGb}QA?l4A+kt)x!|P;j2<7Sx=G8n5Vl-W|Gsb91I*d*=V5 zj{T{W?_W<9u|GLP;KzvfQ*1A{x<)M2peU99??O*^2fqWsQ!%JyraY%!LkX)SuBa1W z21nuVGdV2U=OpK0rB@tFH613kz430}^uHN@xJRf*t1q77GKdC!aQgF^!QP9r5ZcQ; zAN?JBLZ2uZpS;sqrcWsE8_Iilv7%oX@4pvyI#6-(bJ^1i>YrI4tNVX@)QH8ny57D* zt#m}FeU)i~cV~+?Yj2Nnshz~;0{G8Umw$y%t-@Y_t4} zmNF@DeSM5eJUJq4aL{#1F&gl&acaHl9eSS=|2#`9ao#7+7`TDXM@=lGF1m^9f2D<- zW4nUwQou<)Eztvo{o?~OaBomjtFK0ShWP|{rvcX1!GA2|7^F~V=(J}@_5nPbM9KW=^mSO>U}C)6%!%F~#1v}D@3 zzu6rXzHz5HoparjDxU+FxhFc}-7C46F0j-iZlTap@tCVqbaRJ=mJ{B;$|iZHaBb^^ z?$Fh+!FKH`GyF^M341hmY^}4tUgF&(df#ESQOt5Ch&>lli{sKUr_m|P0zh__!SGj# z`+g>SQ(5%6UnvGiu#9^gW<$$_6q^eHP0x!Rf9gXQ;M9swq*LY-xN#TP9kzx=3gicB zH*;gg{smt)zX;Um^7YjEJ1eIHJG!`T-!4=;sI9k{II3@`Vpot$EF+!WFzi&?#9vWPHuD|uju?OD@_Rd|>dMNubv=7TC4IJ*Af3j$8Y4?zZ zQD>xh^bI&?`n^?L`BEG;mdpv()_k*1xT~r7<3U1wub@T8okcMV7XZR>tvTap7qr@FwNJCl&6C2?wCp~mL3{$ zm1jWYFkx%o8pRz84g`~K#tAV&DsUVX350*{L4PE|g%d1VtKPlMw~fz;&{tA1uRC0~ z>kJxk90#n~rG1LSO69ixeML>m#Y?FOlq+nRPqTR;DJ22VfnlCo<=78>dS${pVdJLb4wm@zq~U4LFL1){&dVS+(N}K>vC<9Dj$^;L9rj{ z-m|MKQEuCrFkv$mx4W1W5V%}Xs#Z1Q|J>M~Tw=I1NwG#TV*9-cRyFu8jt`1K^&MO1 zj7i>y2u}Hi)V-X`kpPz6XKhQV_~WdmsS0&Y>(3_>10`5+hj(X)d{FdO?l4*RfYEjk z1buru@3p;W7tw;Q@W!Y;W*fszvhxPclei(o%qm&`d_Ul**36PQ*~2@vD~M5lVf<^~ zHTE)(MxEZuQYEK2A^|s>8`f;@%GSKuYS3qeJ_^_3`M%KXR~ z-Rbde<%-v?U6+VB2JCsiyAO$aAe$S^E{fcuoz2P5`#Y9jbdaQ`c-{Vs(Ha@Q=0%J; z^ZKoza2Wah{l`-yWW^nyE3jKen9PuKjG`+TBz_7=hQXBAfzj zg*XZwM<2Zhc|g9}m_pu#RY@wS)E#ZD+@HA%Qz%d;s^d5=7~kb5sx&yq3mPqvF$7su zF{sxrjkg8SdTVW+(6E7Y)`$&fmb`o(s`JolbyU(o`IEdh`v{Ta^LF60vX|bXR2g*S zanxmbRs~-pH_=)b19(;$`2+@mbJ_f*`LQ1wS9~}0Cp6*BiSHX%tj^4OF2j-exN~Eh zK;}d1PebpAnX*hJD)BJd{8ui~7Eip+xH~_O37iVu`H2^88S;=d>{n=;W+b6HSg~%@mZq({DSnB#)|AQ z$<9|$17eUmgEc|o#vqbmNykZc#qme4ah={5riUHcRWi)CQU$Mt`~v+PXmFt(4K!4- z;n=!mGs}xxg}TTEDkvnamWw{X=p_V?e&9quzLn}?-&n`P5^PteGbIU<14moLB`%5R znrs9|k|T)i>Qhd4H05pOb=(lLBylKsb9A9ccXky#8hV-<0t=aswWF2n13m|wFv8BU zMhbN^z`t<^JRDBPcRo(;8pB5BG>4XW~gGm{G1Ii%wiI-3|5{Ix)LPQ z>&MJkzFfv;oTb3f*&`GPQh}eUzz`%)@d;n$bCQwv#B1H<+0-Tc*6n>LBkL8hg$0S_ zvjfX5xem|!p#{zwI@D5u(X<-A@&1l=dC_~VQ)rEQYO-lNXL7t$_`Va>)e@;Qon3eH zM|?91h5fC%awQc?<$4mYAFkE%7Q~XzWhcz??jCFswf*AOVB6o?>S|2kR?39gFRqxt z5ZlWndgbE4pb)9=IshUz2gSAqbzc_~N2f_Te1D3nB|R`S)4x4z$upi(Mip{hW#tyC?~6*mLMQhMDh>qO#Uq zc>Qsbtny;7Dt%X!l_{lWW{+Zy8B9Yh zd8}=reZ=4QDDHoixn&zSvi>zSgj^X@#)#byy z;u7ln8Z@l2H^wRsEPrp5eL7b(c;rO`l4~&J`sTDk<YMW~zJS&9TCvCoin^QRi1|&k#ec+_)vQ+AOwn8v8~bl5C%F( zEeoj6U|~Dr&`1A*P2Y&O`1qP>4{vry^k)pTrX<51!b-xVGa)cfsK2_1I_adrPh0^-OciF(fA4{JC;+XkT?c zE`e8ZG&L8#C&8|c&lgIQRESnSaN-GjoYUS9{6JV&oO;BO%XG()WuopWvw!dq`Vdm(ZA*&rEGuQx|7P=NBP~_0UkI3JG2oXzF&=R zUsE+>W*+!BLG+)T7at`c{+1Uy$&b15)8;0Qu)6Ll7L*TzhM}tL)o@l3_ z3NOt+aT<<>Jc+Ee_CS?JMonS<0`~M7fG>?ymhN)XM+(#)t^*=@_P)NNr(9TMBA6EO z_PpLJg+)xCx|?pmStKy;I~~0l22iYzUmElMW9#N8UOEz?HSA&-a$zx&b2!{rS>5)e zy9IM?|2F*vnNLjj`*A3UL=j|!409Lk==!0V(H;KXL`O0L7~WZ#s8vfL?5@HLw#PMU%tL&RjZ;BhZb!!(InxTIky0X+| zkNN)gSJf>6gkF_L4DjPDv&X{tsjDD5!I*p$@k`%6xj%n%NS@9FGfcwxLsnKdLVqGK zA#RBxa$Q%dr<`s2?*y3)Tga}ex@0m-{9RX%0m08I&rm%TiQ)#~Uc)25%MoPiE9W~_ z`BT0)q!+Pn=d4C_hyH72IdW5?r&b^xxm4o);k&O~mM)^<%23VHEE(wqUm8~oK7MN( zmVX;E#BSy9Wp`=+0Bj7v4e|qHXNulL7AIbw0fZ;ayS|(?BS#C>4eRFmN;w5@%{0UP?fmRwE&wQAYG)~J3YDF)-(Bsx%K zP3OobUSADwCwzlj8D_gjsv8!gg#cq_eP1cXc}(ikmGVw`C*zV!L=O@-kEr-J3{7yoeTUhJrK z4v8b_vfG7Ko%v^D5)N|k8AAB^U&-t>gp#kW)@v& z^OpvduaNN@xgv<~S2E0f2!57h9XMP0SRLiTbN$M4S)KTLDMgRR=!Thqj$0zmb4>mA zM_2C-9SkPt8k?d$UPSr^JGESA%>ymiDeZ*{N#r>b4$tOkRpW|z_p^vdP!*0Rk)~ay zNZ+PJ1yCV6^S!W*X}D4$+}5&s+2l{?!odCMimfhXgp}XpQ?c3 zl}1vF)Y@Ff_S~)Gt}}%#`I#5Ea7ax$Jks*gKQpgc<5*!KTD?1Sf23_|;#GB=xK zIV1;KcYIvmanrBJ3SBGHtY2>C7J}iZ-plZb!U2uj2+NBq!9H>_7eB1=u`$`lDuhts zOcflor?lR+d%`Lj@8hWUzQ;gQ7bjin_K{o|^bQWa_}FUGsOHi`)ZA%%-|M*5FJyK4 zL8A*^#D=~E2}=kXorHS6of+^dN76%9%iMNC2Gs;BYNeB7M^G?1AFXqYH3)8M&78bF zV!2qhemc=8TjU%2HO@L-@?Z&q?0S{uIud9f9E?q?S!o<@!+Ahv_LWT)luio3IB&{s zLCofouM+k;n?b8+8&)V~{5NiO31wRuKmV_;ONMgmwO42+pkBSuwiqD@Tf6UXi^hWS zM7Am%-I+A=5NuCCA<`&>+lJExuhy}%$#DyA)|t9$2W~V6Dd?f^Ljod8d+e^73mDy? zsm%> zViKYWXL!x+E434@9UnXI8!Gnq7Na3&W1hFWx7EFVXc|Bs9%|tit?7S2Cc^iy8leP< zd}5X1XI;mevTLr82fZ7=kR@?DmQBFtucU7!^C_+$FHdLN%g)ghV9i|T{BoL5jWxB9 zEr!F*W&6%@#YQlHpEo7S{L}48mkru=MVG~uHHJFc<5tmlevR{^ld=1#ku_aGcc(U; zY+8M?Q&;%ORjP_^gL~O@-rBhzbvA5r6rO=Mf_PdZe0pbyMMFTxn1>!i2U8|pZ~O6q{3L8(ReLBE=PxlvC$uD?jE8P-^U zh6cqRTTfrIv-5ka*Bn1Q-uI&JNat2@MGEU3;kEy~#Eti@M1~17rE`8EIAXL4&Xfe9 zRDi`+Pyv$@uq4|P2f^nhEFKD?B-?xR=T!3q;gvHytU(BWs@-Dm_)==G(?RtFCF93&lcr0>)Td5){voLaZLLh3U5c7r>hr`#p$|DXH4$1;SM=KH46kEzLh%jIdQl)N zObe3?9;;`Tgh?(p-NLa1riN#o5iI}@JOx=V?;!3ls#$-LskZ+VhvWe3LUcVjj^zLS zkHn0gd`P~u?nxQOM@vG#+VQPihRFo1y_c`hRqEPP{EFEO;IjksdjeW0D(h}) zGtQAmcm2|mg}G2KzAXtFkx}N3Z62dXz!LZL+U1%J?=3o zBB(>^?$MW{)!MdSabjLAY7H(LM3!5^dv` zeyj-Amhsco3lbCo;=oX3VJ_h@IADSX@kuZ8@x_3o$9UblF^qN`qH8YgBbqJ0(kWg1 z9JZYmlvp~Eb{x1)RA{xjA@s;q<9P$;N`Jr5m1CjE$z9ChX~N5}-A}yM4h*e2%at_- zv6_C84%gl%CZh|9Ku4)w$M%wIl5ZQzUJRZd(`>M}?cADa+@jy}l2hBa%u%Ri@*Y8R zp_r|gh)IsxH+`AA<2XZ`g0VR%+1d}#+Al^!LIxGy=`#C5=*aBPA~hn8%7?E(TQoue zwEjOqOKy)~3LTcGmOB(lTg|G%Y^y5jFk?Xh?BHTAxVp)I;xxKQnvM-*Rfcwz;~BrT zg)9FyF^fW&2jKSBU$GClGPvyHyuJ2k7W<&n7tX~5w8apHFRC2TaBI$EN$NIK>hQp& z#1jlG7m6Ms!kPdJsa4GzRcDKuu;PUvhQD;MKs~&(Q+$1W2hYX5&(@Z$o%A7SvikJ1 ziv`aHa8FWmTyh7Z}h*7cw!w?*fxpbOR!MM7nB zpI;1jV=UmDoaS_VY|*5?0%CnfaVdLj?YC585y#j`j4h{E3`yP>h4g37Tm#hnQjrUDk1k0% zxdxs^_$QNXP1OZ6%?lSid0Ol3*dd9a?!|MH)(oncBdb!x-RNWqo*cb9Mv=BPx}^ie zETZ>Rd8!)1OqZfV#yR-pYuZ~Iw;Bu&G%IssjOVu4vfymkn+q4YGG^qg$uZDdb&zwV zSU9(`#|P2x)*Z#7+Ex=MGB_r_x8)EjYc7Z%?R}}wQTZL?*Y3X(hoPafoC5!sqvIGy z>e0!IdzX`w#Atb|rEj#OtV*3Vh8z*kTR(W1mvEU+f%C8_BfrhOtN3VZ@@eJGs`by^ zr%2@4sxOJ!Q}q$(-wke-j=S>iEE3~;J@AWV#I6D@@1(ihb{G0Ii7qsqw_?81meV)v zHpz1*E;|_S6W&p_oM>B<*VY_IU{AN9Y)yW*+wL&8n?x9cBmY_5Iw({4$ zCNnG6kIvIXD7W;^{KoBHJ{1dlAh8gzsjs&aU)d~^td*!FxfVX;S_Ya>2l^gfRvq1- z)K^08{lP_CyTuIdO^Ew6$ZYidJ!=Do9v|X>UN@?N>XP#V4aZD$wc&W@xb_jZ&YQl2 ztaXfq_c(J3LAbRiSz}%$e$egP_r^-UMh-zgG&)yXBfA*dp-0-08!>v56&J+67vWQP zZG)`BB@+(`28iV_l{q_KEEhD%4UtBl-m|IhQ&t=pax%$idZ9&;eNzeu&6a5NyT-%0 zq!l-|9RV7_o>uk3OXu@(9~~Sy)S>ljd9m}eyt`Ysf6lmORsS*NPU+%>(YS!Q&l@>s z*XTg5hE;L9NyPKq?kqv;i{yoj`=TFPbCrVs z=;p|{^L!zW4xJ#*1#Rp`+^r(p;5q3jGY&D{7o-N;*(za8HEsjKFW6StuzZ>Om|xI$ z=Trp7?c;EK>L~Ij9itnLF}?|3eruK3>;H1c*CU!iH7bI0Rqfb8<&Ixi|5)#RN41{r zc$TbZ-YAz^h$;u(X&xQ{yRNLj_0!Cr>JpDv@YO8|m05x&cyg<+;o^L_w?q_1w^=uQ zJz4v}u}w}mR4d212>zmP{Iu5oHTqSe(LOl1Ql()cY2ZqbGMq_O#bIWdrk#3oExF3& zF8<7YU&bcwM#@xMW?ITqKd?WNO*D2Q4CWrD!n-%;oCkD^^fS_D&YnH%X4X72P-ebg z&~EIzzciO8L`*m1RTy)2aj@*e`rr8nqzM$_-{@LAdMs;PQ%4Ic?&u_c*N;tda4b-d zAyXr7T>DU*UuCc;@3}4Nlc{p_}I7D=lLc)M&JZ;u72Lu6r9{{ z$Z`C(O_$crztC&WK*v!3VWr6-E;xqFED<)l$FNc}ycKseOVKF)7qwwl!cD%N!?}_J z-+P0tkPX(v_|a31bNpCtu}!SV%|s zLYRewHlYVEB(WjU)~%F%*g_oo?A}S71lASK>Dv<$>V&m4mDX8z<{2=OG`Wf}d28*w z29772ibMXOI$c}TGkPKts&p9Gc;g}OT^NX_Ef=tG%j|y0`X4FB7ZhR~?tEDi#k#3H zu`)|DuOGy_et%yf0+Nquv}(N(Qq_GnR=fk@S=cC<2G0>?A(Oj_Af`WGfGqV%D)`TN zq0^1^+#|W5QDvYPlpIcKS;BlHN=oH`q*%OYYG9;A0~K7 z-l3B`!}Uw!<53BvkN}ujiHRGx#CkAIk4m({lt&#T=}V-8Jfr_QPpL>J`M@<^ouu%Y z90Nly)L6IlUcE1XJXJ;o(yyN@(Hf5BDp~k5` z^NQ^;#HUA%cBwDZ1do2WZ$p_3P)^43S?JR#_KU-!JuS^%Id9YiJJ~e*qbzLl*1ou zC%gOUJn-cOd3atkjBd0YzU!c~gMFsZFSKMEkxqoN#0|wM;zmGHrQi-4ZMaX2(~WCe z=jy{UMLs2p`;i~IrU!5J(a&bw3iM!}zV!}2>NVONA>}`7!GGX97py}n{8BFw^nlx3 zEqaX1jQN*tGd1LhuOuR$REWuZXI;A3^qLRBehg||`;89l?jO~31VQOouB54A2n|Sm zy(-oKO>z`WHFua48)Wp}RA4{gH~+;(z#!MP^GdpIx&%qF$0j`FmHLE!X%!|%rsqU* z^~m{sP03XZG=^vDLT{*qg?3(SRXS+%q{hy~iV`L`rtu%mEYSfUm5? z+c5aO8wy&-P7`{h^g=8y)0?7SPZ%^$6bMkux|8Jc`N;*87~+G|;25VJvUzflgcBa# zwhp(~TCD+eOrnV=_VY{r0dB z2)-_To;JN{pGjvANx$vNz4t^Juodf!R^X==32Nds!Sea zUiCc1Fx}pf?3osxhTz1tLq7y|ekXLnBP8HQSfaROT`ppDQu=nUg5c7oT=4}7-9q@pSnc7ZH6Tp4;gO3syZI4#Q3qH&C> zqy42O#*nlS`u;;v_Y8*49_Nx$18tz8|GtfNZ=OaEUyL_!ReTT zH&s)r96N*~3w>vLOFt6i)8HSuzEuZWO1czYR%-S=lM>E}Rwdc&YS=OOZCAWj6Y{k0H^$yrmXJ;Xe1d!Td6_xJRdBBxYDMoP5Hgns|L& z0EOB#=|Se*&-8-TVoyq{%eyhdY4NW06BE(ib*{g3Ey~QqKGvK~`$G}eFLtW_1Z|${4I|A)(H^6?z}Z2` zKqv;BzxS6`L-v$b7(FZ|b{t;LzC-F)e~~3Tj#Zhrc~d8!wg2uI)U5QnW*UT40I50% zO3JVO6LP~b<>*Ib{F&SYKM=}dICw%e2`A_9l<4j?rJ&^$sAgDvJ|FCXhjL?UJV(B~ zdF>>4ut%?U|6r&n+>Ha|(oJb(n4Y96MDg)L2ssv0;G4L66b|shok?aDJ$CWcFKRO1 z*5+~E-K8m{8{Ob7UHKS5hf)CM>v^eCOwgZMp^1)U+2t`%Y?x*R;R(Lhl#vQ+*&9dB zUgH+ej^>P(`yRl32CheDBly3ny(hHMcQ(vZ>^iRCF*HN*_26M$E-ENk5{dGXZHpv2 zI$lh@Q8a`!{5mIvXea%lk6#dmAkye2_h+aAMb-1Hq%mA9eI%l;BhWIjCaN>ReCSjP zyk}{%;AvM9ewBnYb~!?J`s-PVpW2XAFWxyhC@4!Kq42^uYl66}l6v>6wm>qGVPW)~ zhRufJ^Ig)cG}`S#LV{iu(A1{af7{GBA(;6JlI-}?9;Us|os z<_8pT{Z)=^H%v6UHKoPAm?oj4^5^%D58ZHV57~zN&KUc{9dbkGK%gc}=oYBgi;KW) zXakZCeORJ2Q6SNAAgMeClgkBoj^5N#@SoJN4{L#bv|m?LozMsYYN>L&R-EXvIh?Ge zZprlUtZ7Kwsh=AzikSCuDh8Z*9CMJDm-qW8SQ<7~TU`lAmx3(#oL>|K4;@8g`uqF98a7)cTDeQcAD#}&iBcxD6$X@YLOog4LPoT+PPi+R&R0t8z2WHvc zY6A?OQ*QN0_@~D5hH#Hjf%~T#Jz^@{K?L9J))T?*oe9QA{kz9*^qa5f5jJtb5w^{G z`$r9qu!6CfE@3Ui`wp4(^7#Ka!nXm$8u&F@WWyjb_=a@cshU_{uG^>}y9C^1B)iWn zDWp`j)C+h$xU2?!&Yw>U*!d^I5$S&KG=egH^9bD5ADrviX!}+=YH?V!fGg;xhCV#l zL61^jD(T{5TWlgu&KO=er|}CIGz?yyx*nx`Md00hD=y=>NUSZ8UE}qx%Yu&zBrBU%X(ZMC~{s+VcbjauyCs9(OJC8KEea_F+^~!>HJH@ zF>=h}r_+INHv?&peh4GyHZRm;q_^;{0%aw$?n>L2yI(`_Ba{95?m;X-2a$nweky0@ zAij>BmE1XqWauD1@52@A!+prWsF+;c=MhkPvMh501h?1%^9rkzFv0(Px0rY-PY*%! zLzmb3#RfOi*?lNfw++fqJvL^OIq%wgCL5!86wX|p^vydQ#S$GxjpezRaUNLyQ!6V{ z|8eHp4In)Zeq20GN2(3~&CWD2(16UBtLrfcqfMu{%ygrz_2gGu4?T_y|0tQ+iZjoOxw(6nN!8&5s@)1W z(yh1+XW~w*Q{4_|+Ye_jl~8+{h7wj%auh%!DNgy4#Xa)}hjfx~l8N$Ap{r?w!`QkUdw{d0gf60gfE5mestLQ9dXK__&AjeP=KEGo* zy-E@zcR-r2;rPyFR1N=?%pf5Z=ZY>n0SUd3<}L825s>tiwmpMj$;hpTZd7@eGKBI7 z70AcM>5HhK_KMHHIR@FzxIlgXDliKa1LJ3ILx^Qt%Hkq;KsUN|$6RdY{4=2*?@aM? zf)N4eR#PiA9Yr51gE9wke0d~B5gn>zwxrMgVh~$-a$X48_rgEAU+@InhgWYB0M-zb zo~PKa&Id=EAok;Zf#w)*^PISwM|pOmirx|8oLvw@dK5O*elg&C!A@W)?&;`7U{6(_ z7E(r0&#RWL>@{tfGK!a~ZmuC%ff_Jjr}>0UGFW9YA+eBY7Tu zYB(jPQg5Z-#PFw-EIk=H8vkABW}FgxtT@h4^{eyJM5*9~I(lC*nWhw^5Xb^^~06nJV6 zbucBu2egz(ync=GpB?1B@V!43QN(Sc7L~Ws#cfZ6CR#LQ|H==ZnlP@gk`@#Hgdi>= zkRg?vvk@`BpF8{M(b1zhy!G5O0ClbWLAQ09>`b-#l+^k5j?VsLzY9D^nxx$N4~8&J z@QqqR`dlvmhMU93$R&+CPw~+(!;X%(c3j`NTA^rhc0iUnVueq3SB-eUS{OKW_-XkXQXi(6TQ(>FGtxg|F1$;Y7O2tVc-GA@o(a?=W9ck7DYzsy7VYjbeo;L83 zFwVS8>!>tLAleVaZkbHCA>9IkV?m!TC0)pC#dW-?cA$$#q>MMO(|#0NX*dZsB)5R% zcFS;jc}>f!LKpyD_%>RHFm~VQgq?Nx0+<`@*M|m0-^4*#)(!OXZ%d=+@BrHmKR9;t zY4Bft@Ko+d4LPJxXdQoVXFIIfJj-&_J}IZXt!XzWe8s+zG&5`)^8U3E)r#{RN_#4Z zFun>sRHv`sPYp(*M90E7y_&Jl!#!}~sw_oYFt}Zu;e7`EFTg|ZDhnFlBu1j3CJuDa`C2*Zk%y!NT zgVSiFQ9XnXYziR;zpYEa>l;rDAI0J&F=j#t?GdSN1NX37SIsr7tg=OIn@(|=zKuia ze&F6oO6Mz#A_B09Cp{wGUcbt4(Kw6f*5i<~o1LYHnbdw&3sddCAipv*?uS z(%>Pui|po@9JhB!r3-MC-mBEgzE%OsFWFu0+9Vy`YczcJ<3k-ddUx@c5#Ip(q`21s zXLOPg=Bx6au>OLA{6+VUJZr%P5LY`PSyPBFe|x8+oqWI@7>yQ>4`<=kl%5<}LotVM z@AT|@;puN4jaUlFWg_1ejW{YjFb7etuC)V4TnNnknyApJ`pR;f1~ady6OCi=FBt;{ zUxkbK=$@AdHPv?1J9#4PN(7vrO*^gi4*2xVW0t(wK~TAH4Z76_cJ5g!EQw^~#5ulE zBG}ZG=6GCW8A{3@aS(alxIt)f-rm}@BJp_gO#pPhD|U+t*_9#d{wPx7l!A_oN&^9! zBGKlIAS9~V?{n|2m-JolLS7C9|8(G;i+&!vA@#Dv=uROJk4ZHd<@#eO@?l$-z=8BI z7w@gU%DGkBwq^i4J3|L~;>{L$0{mG;FlLZxG@iVFO74bIaP8;b z1Y*FeLyd2#M!1$$pE}Tc)$v7QDu9ZYAa<7GZN@eK+7XQB3ZJ~E3mkZQ+RLg#>51VN z9{e2%vpx3&F7gsQYJc{u2yL%W`3VPB1+3SWz2NcpQ~S}~Ytf%|RKzH6E}H~`LxS?; z=%3uH?6PFenKdqYrn`Pv1(xr4mgi4v3h zkJ{zAeo<-pL_OU}^S7NoTh{Xskg!sw-te|eC*Try#1S+eYu*?!1W}<;5%=e_0DSl$ z=Gs6?I1vC;?|s5$h_a4wC3SLg(gi?SV?Np_8Mx$x!fr}<7)mgHOl|Ej28J)`&Md`eWQ36Rbz`RdIXcSKqypB2`CO8iW6OOx1Zpy%!sJ zb6`u|;!3D}PXd$c&{$%ZT+GR%4|8oGGF4vT`>{xdYH(8I@L$po<;LDb+6BfF)>N2J#LM(bLAQ!31 z9;$Iq4DFLN>@3BwgNV3i=c?X;C-Dg2Exws(;2U*EXJ#mL_USq)#@DA=XxhxsItT(b z<%iaZ{ul8TQ6E=?pvE{z~f`iFA`aiN?RN)2LziW>D7TG z>9c*B-qR(v&ziCs7aoUh2C*t%sz)TDLd}!z*?(pM@GISVGs4O+jk(jRC`${aP)>PE zQ+>j2DI{yzvDK8t!Me%Q6G5a7k_3Oh&|=Uv1|Taeu5>6Q>xfLx=RL}-xZc{xRJUT+ zme%rYr^+4rr^-Fep;tK~uh8MsEmqeF9e1~f<(bbt?MUXle)?$;79+RfdGO1=f;V@P zm4$(tF&vyP=IAZSHEBN=*u*IY1?6r27fYv6wEiI0xfRFzrrz6e6^;v~)|Y?@%D}N0 zW{SW9I+`)n(Mt$VkoLjrkkx#rBwxpPUE%gnt6@8Iy0K*Vd{oqO3UfBz03X5q#I=+`qYw~z#@@s-)3 zIzx%Sil9a;Jv+hUGt9U8-r6&!JfHgiS&kitcv*$zE8`|eMl6scvExM>LvXzj#mH$t zloRS{QO6r}v@$MM5#!HId~ zf2ZXri5c+irink>kP5d1us8`v_&T^j4iY-y1sH^kHMx52+4{K&24hrC38Z$_GrB!AD@hmsWxEBz`)@nsgmp>?xqJTb7y$rKP5FBV!aKCE`XRqvP{m zo)$sZBvLEFX9AcFu6_Lg0}aFgz|v0MAeugo|MvM?;I)se161}@GYn_WNx93S9TwR3DcOx~q;1H)ULs)V5+?&+vlNpDFS zWznl>yjLYOKpP}fU49*vxwpCA6dh%5JM$bcoLIgUt=3^t;1EZpAfndyaLp#oU?HNy z-*+_=jFEHw)|tw?G=4=~_`{HGy;sX%Ky*LM#T7=GANKR0K{_xn`(AY1=Y=BS!;NST zAM9?Ca7Z#yFCL(_@(=qv>a@{L zZ`cWKc7%zG;D{6|ZX;H-i=9C`_Di6jcz4GN?D_OCvnv(vE^Y#65=r0U0tAlNy+BB9 zjDm0I`1QZqVp-#EO+#DvVMQGd3ymN^5_dk8MFuebkZTE5;r2gW?^RI1(LjI3#Z)00 z-gs+A2%_MZNtootF&g>D+}&EB(Qkg*qM31)U(F$M%M8EOvk;GZ}FRUICwx zU4DW81W~&L&04he6{*)T{TC-T+jGBvbjfF2k^bm?^oU1O%LYBkm}bNE5>9f>pEMJ< z?!&Mnwe5Isx&=lEiECZCwUlrZhE7!T&{ZP8M=h^&%uZda1UdM&%L`egMFK>QVw5A6 zrA4L=MX;fQ6wO#Y7%f#~R$O9uNZOX}NSZ~#l{aAT5USJs{)^?WGHgQ7NbGmg z9qPwaYU3CafkNmH_1_%DRTDC@AJT^_29v=a~w0&=Re#G{4*yaU|)1nE` z(bQ>f8{!cX2HbPcbRxtKFJLo$IPOpPJH-QNtrDAFib%2^FW7{Zxf-S9z7t1&LXR`xy+s;HD zBGp$r34KC%0mDA2o1*pSztnvp6|Amt$9tU64nqrsP`$T3F30lQw{O*8U|WSYap#56 zmMvL~&R6-p%;UmituR@LZ+)L+RZfvqdPn=JRQeYXM;XMd`cpbIFYPT@9Q`Tp^`b2q z0(>mj)pu5`8yZM*sVdr&Tz(9s4lY_Vd(1)!UI*jkWn+2jU*h{l!a?9|Oyp2_zi@!c zLdfWLG`DzWJ3#9FvXno+Jhcwyc=myW`hZ`L-tb&2u2-n$0U@tEVQv_FKR1O4WaIN9 z4*>w15tjC@ukXAk?$o`g)6Zp5>Iq$N)v`%~raljFk@p}=I??f(;jiA3f_JoEH_ z$nxP6sIAW#0RsPnL~tn;|3EW)(~^+4yw>aRzahP}#} zZ@W=DB%-2zl;J)#-1ba(jPdKk^H&;dvZ5k<4D53rhnUdLa5hO6iI7BwMGb^~@Xq-J zNxnSZPZ+q>nk{hl{cB-qzJnRA+5T!2-83r`BiD39gk*{y4$ew@3XXW^a3D(gFI4Lj zN;y>>DE7i1n#-!y%rB^M@YZf3kz_CUYlxoc*&FPmJHr>h1M_n+5r)Qx4X*IgDV4*` z!RG1=oT&G*pGvZYzBzO{whke<@4o&m*mDPe z6t@LPk)k70I+9h4BoPra@D^)W1>d@o6(b7rRYN&MKXaa@uQ1hU5~9)(Q$oSTFHeKn zmuB^|I!sE9`6ah24wQ=Z3h~g6`!=9zTGr@&vEVJ{>94S>&M8r9iQ%?MB7F_bEknAN&X?3UCVU z*n$*~aQaPE1jJtB5Oty!Ph{VH?_^oea|Z)Xr65yeO&-2|IB3~5B-7nnoYWUz zT={CasTAxWBUybJL|A^XZZWm?!vBR#Q$BEM@~w&wA1ny3h2>tHNXGIxZolooha5ga z=6lIepQ=3$W#0lLIli46gYJzdQdrpi9mEm&j#Nsr2u-m>7bmBK4!L61wWPS9a&Qfm zUvrRaz-Dnc3iVXwR5pCSqJxVF8@X$6Q+VcezBzG>Nif((&$5g{2ylk66xHWHN=;2? zLqSKLYXF>pIG*+?WO*b?3t82aj46pE01ea$L_4LHph50Im2?D4H0AwFsk`Lf# z7nwUJj!6);s?iPI-n=;V5+rLbL&e2tG;1#Q8t8`C43$?kLpOh$BVfx1Cj*Im$X|2B z^UEnc_S5kx;Zbu>F8pu;Oxv~X)y>v^bJXrYZUG~UhUl->?MRymm_QycF`E3~+(2>O9 z3?5^v)4(&NKaR_^*WiO@)5ynH*kSF}$n2Wm@5oWyJTLuNTzF!mjrok<92af|l4M$xkw_fnvj(5**#X zu|XEa21wB@mE++gP`Gi+5^!7bOrkPPvCU=OmRt}<`%)2MwKlOr((qcR>S675a zDVx_5g?1^LMAh5l$ItC_qQ?eP5a zOL)Oo-7LZdoEr>yvNC(HJTHOcT^Qn!q5eoyh%-RlZ2BrIgU$A{EAS+?SsJ&>*z)RP z2WK~OVT0F;yZalb)2r;0PyFYE>#+iGQP*^mUd(x!CCxOrID5e-8exvDRR;}y`^EbA z5DL^E{HODF0(BO)NSp0h8T3S)Xczh(&#e!3urgIqx^v;WOzPGl&5BdNj(%`q9fw%X zDok;WMKPd~fIr8@MllM%GF!FdzTQ6kyn@22} zcXk9c1+SoONk*16??l+CxA$;qwrGQxkNV*DHb|8)4*`OiV=t(7vtx>?E}?;FA~I5V z-;HIN0m+aXSV$?ycE8`3YK`(aB{Xj97??)Ob9uoPB#c#^>FYG5ctY zrj+Ni%~~I+(CFJ;6Hs_*U!b?Fl!wajU>r3k)gRrLQJr+^ydf-LX+){(>^dr6!NKu^ zlce1fc;6|3o8cr+5~23jkE7yfy&UcLoanaKXpFBXu;&_Is2CQEajgD$k&a)VANE~{ zF9OeyRLU=vkPl$k|zgN0~3cayDXCm3Oz`9<7(dw&Vb8LF4~q z3;e|r^W}WCX(vYC$D(|3Ui=dYREFz_-Kg$IVcxy`7_d-9^q1TGP5^;Dh zu4#f~HfjCGF#^W>UouRqE*JqtF9#0JLXR%!qvu2O3gO{l&u5Hy+{FB=01Y7B7`;sz ze`^MfP`OM6GzF)w#mzvVtVZ7P*xNL3m=2v9OzDWOLv`iQpLc)1Ag_Xsq2TXv@b+DY z!iSAm8NAYJp-1c?7NQ{BR)EbKx(_k|L0U}+t#I+F?D}V1ri#^)>ez>fxX02jRU8KlrYn-F`A1YCfsH{ zx&50s3P0U>xU0|5p%xGJIr8tJJ^+r6q`1WA_xAuD6b*}_yLksUOi=}9JW{def3kM! zi;9)q)RT~3nsY8*2SD=IC}FkiWXlC^I1i2El~07~U}q87^|N=i0dXHGrwD;+fr z@KxnF2vjKRFJ6tH_IO&6k#M)Jg9nK@3?Yw!%=R-YXxvc}X`GpsfekB(c7$(HdF-?Ch7?UyI2sw2} zn^N!S(RV6j?8Op-q6d`sb}eEb;o__cvFQtn5d;ck?@p>-f_@epCtHRFWS9ji#B1sd zSc-amYYQhJT7DVtjSto&o9fCSx5N||YSE*SbTt8BmyevzIXosxcO2O-jTJxry?r*y zS+CguPQ##H+J)3zyr$}dZLgM~B5}v3c>-DoWA5bFeSqj0#6_qzcEGv;i-mFkLo%Kx zOF~DBQ0*897Z?LA#N0om%=h~F?bD+&Fex;rBx7=qfV+q>g;R(~_Vw2fGF_QsW$~6g zPno|5O4u8u@@QdXACz!|%^Z=Ar7(2JCsGM_Gl^5>kjnPP!`Z4C?DVH~Yy0B`rK^FSD zG^(~X*RNz;kZC)5tUeF0pN%I^H^&i#8>$j}>-K*D+6!p5OeB#ww$IN&MO9qXrUgnL z2UqiBFwi)IJ$dZ(yS$H!*2O28c<9O`NCqf5|C?xC4m}9bn4b2;Y+V_C!f4?9a;2d3XNxvG%N_OqWI~mbYYO%e|^E@~n?) ziIf}17`%6*7Ik)BlwPRl?<$ZGd^G#&?d~03$q65b@y6RuoWaF=gNB_dagbq}k3yPV zigw4#=twyE&`EJT{zNM--iNF9@qE*|=-bE%qkQEM!Y+e6Ma$bZ%zDuhIwMB$cF30x z1>!pw>&qGe7!v!CX0d(XG)ad-?Z?Bm7ci6T?yz#9_%av`UP1@;vSsZa9ANg4ppjNu z`+CO*{Mi*2fVmh67(6%}{tFr7uE^UGr07xqmZXP0=P^H+k>Ol#By%UbW9sMFOF@gox`e-f~G8%cTK zI7(cID(v?r4rDlT4pOgGXwCe%)}h36D=fP%axQ|SnAY7?1$Ad(y!W5EA(v9+268nD zN$4FYDSMSJlr}oq?b-^t{I-zC!R|j4`1I%xf!PS`V{5ANea2~ZqUokbJx$`>ZewXm zvOGImM!cY7_{v>2y+Vl@7z|>(##}Z?RQxdv{2{T6;8W0F`!jbA+62NKxeVl?0jjsau zZ)MC6PCpf{`|O`nSX_K=qILdI=!CFYdCAl_-hg6Y@VZ8P*DjHsd{|Vv_KJd=zl0R)z4MqUdwD4CJSxHNNQUCVbhGJCT%F0R; zZ&B95c6lH?>~$q4<2k!vZJ*O)C^yRmEtgdN!s?IxG21xPUv6nKioZwvQ{^5X_(ykB z+I!OIp|Bb=qNA%wJ`b%C?wDjo+3)!^90`=pg>NA-9If@j5SMK+Xrx$IKi^D*epXH=E z!QnViU~wVwWcROkN|>FE9d^ud`s)OpK3#LN^)F9Rda-N)=8rm{8{S5#oybzabW(Zf z(K*5TO}=b9AWjSa%+gHXx(e-Jq#oVYzn*1Tq429b&;L4!UHt?t&Z#`w!c+C_2!lfr$0LduKH-Gz^+a=B^}Ihz zg%TZ|Y5#nL7dycP`%8e*VD3zAu(v|q2cj4$ZSpm>gO0dV$*K009kV9Ojdd|+CLM6( zo%|BE9dfjdLZIjDNYhi8CDGQmKCQmj2y^Zuq)KM|K4&mtc3)x8LoGWjf4>ii7+(}s z=?Lq_zm!Ox@DK6%%z=Qp|`1*d+w;zEp!`cb(yBbH` z?J1TF$>sW(70Z`TTJ#;bDpht5A!uQPpQ$*7vxDaBjf$H3x z6L+4D^(KwTKcX&Bo}-&C`K{4@1j?!@Z51V=-;dW49-o9xV}=MSF7=QF z%MN15bWEamM3bG~Rp3Rg9aptsQ2{+sphqrb&9AzF@!~odo@nK{Pf6C=9a}9>+Oe){ zv+wK-^dut!p*@Hlb&C;cs?iu|``!))+a&7y2Ce8QSY=QBy3o8v^x{4>gwizE4~)z6 zaaVwkIKTBX=A8H^mz9g;RKwHk^tUVGral|0!p#ZRw@=b<4bI=u6TUnwc&7wKS;JJ0 zXVcs4mibPo;*bYO{{WR`$?2z0RC8|=XpX<=2Oh5{{tu!~C0-^*g|VZ?SeEs9Ace5;4=F?5;JXmzvOr=P(p3^Rp`9ia!s8{Y){^p4OOXRPh4B*JP;|jaw zweK{rx}q9>vp3;RxhB1Ic0bn8uk8;(c8PS9h#8KPW zlBBBdy2wZ$F#m4HIUX$I_oM@&DsslLMhkc`d;yZ$XJYv9YeinbLH7&T>pzbI>3d|- z^7syYDh56dw}zGZ@8FCuu7|W&_EAj*afkSoGU#+r%U8(w1`W_N3^C_-K zmg_ihN_{K<39JheYw-5k5AmNW0z+Kq^LH6R6HMfsN|4*W*>YaQxMM^__zA3`pmQYG zHiKsTe{$4Y`jwIGOg z_0xohY)vi81U%(u-qqD9i`)&Y)htRAhU05yDaATfTd86OVoNA1nK>yx9T()hqYUI7pAakA`k_ zeQ>1?h=$(X2jtmWN9RA1Z-8FPS3z)ynZ(M{FcF>)|o#JnAk z4CI9S$uJz$F>-2E5@~xeLY=Pi$u>(hq4EMhM2|~n&8WHr5ow*kEl2S$X+4iZd$5dS z@?LGj#Sme^^y2T_P;tB;%<>QhB9`jpBe7MeHzQLs+EsU}NeUXJWwIE#v8#;7Uo?GqSC}0yL0gz*AWXh<0;Vx?Bo_e2!P{@3Ow+NsGZG^QVtE?CGxqvsqji8MvXhcy zo?uh^ogA{$akk-T-Js)xbZ7r2FWH>d(#IF)zj#Z}ncF9k?XQ!hV3pE@jtp)vfSWvg zWJsFk-E9DTK#igxj|a`Pm%&)H^QMj94454~t97oE+EYzi*0UBvHEIm{O1#Eh)Gy4m zCvXuEp$d+YQDW>ULY95yxO79rtI%usu>5-qzaX;k@A(%8{vitw5-uJ}1pmE_ckUKg zi7<7ngB(zj<{bR-P=8kvJL&U96vU2z27di4BDG$4)v+kvC+UC^TayV>{+dk|5yd}% zLK8su!IhAXIV?q~+Ky28@E@VDUHAP1TX7iM7B?E~NOkc`U@;hfhjTeDe1gU$M$#>g zn39v*!S|HgUde$)i*DoZS5vPj2@=PN+*Tp-^@pc>RW{C=*Clr8*;9F3i()CZdgNwA zdA&8a#H&hTkGwwkBQ1ym*o8hpRzu`$E-V?7J8y`n3i>k+f_q0H9J7Hq3l*yj$30g3 zBq2n2Q;+)R`()YS;RVaqo1e-v>(=z_lTI3oy2$R4-1PqDAq>;ebvh7aXeLny)-2s* z39tTs`(7xxCt;M4QC#_@Q0VpQgEk(ZFrh)C!vb`jo$#(wd%=wl#`0W4R``1zjTa#8 z8-s$T5h|KsN;_S0UV^zL>w!(f-7`Y@fV8dwYG+%TJyx%U=;E7Zzr$>D!&XLKyu=lJ zZq0?y6IRa!OLV(vO(8r>H9}gu(}BY`szvAzk=T27*4GujfApC)LB5pp@Zb1`2={_D z%?Z0vgy$yozIDN8kKfkbX{YkR?HV}S3EmMsFwUa?q6_|X z=sdd>?-Ti((BK>XZ9R-uFb0z>bWd66>Ig6Uwnk%P-Hg?;q6j9AY=c_nK8xREb+C-> zM-wQv2S5I^91eV-e-o;8K({I7z{HE1{p3u#pqf#xR`D4h&mz(WZHSpPui2auqQP}E zV+1-v`$er)KJ)an+ybR*pr+}5;xB639NBMC4>hu)!WulAR?ncecxWHx`iV?Hvg&5L zxhKT|lRFr8OftF&cj$i-TtWB%|G+y+?+>SqE&W$ z;r`NlzVMqYFX4R`jO=3!`3gsj>fzHmPO{Llmi2a|lBDAYbO)2P(fEErR!Df6AYm+I z`0hK9qBQbsu(DXB{&nBmdJqSuo+11V-o%qxQIF}{@enNlj?_|tvC!JPT52amMlf)V zE^j%KaV<~L2=y!$S;`&EN7qC!c2Z&wiG^btb>N!Je@s-`>+FW6K)S~gNwuRmFyhNR z_MfvVImEgowA0-dv3pGTXhP|-Z%YU&`T;mu8LEZ}CO#_*_8KJcEY^WlX!6_dpCC|J zEkR-Q4wlp^GK#(q2*op+)H~rEi;(W=hkN^15 z1!jy#foaP1t}wE}nXxVw8Y(_lHJM~huvnQL+eyTuCWHyleI%rAOLe6cXPcc7wS+}! z#zB|rA1zq7IT+=eyjO#cd-;Y`3#r4`#cea;ylWLgshfAg#sroIdnJyVgKwAYR~tN3 zk_{z;wwHd^VUJ<_w}G->fL=sGyro&%@L9GY`U}96R&R9p8sAoH8 zN$m}i4=6dI>j*Yi_U|tm zVK}Hcn%a@k;T{M{>s_C}7T1CI<84jvSuUh*76r%HTB$PjHZ=m~bF`{kvB;hF(Uk<>KjxYPbYmWR!&j#L{2yh#H&yX>zloIhxTZ!% zXE5}uKi?S27BN4B6vQIn;$oM~piOuhnR{rb?uEbrjWkdQ^!obx`bFDZz)m_@_MoY6 z>7Hrn?cuI_&VvjHQHt6;F)gbAOKEl3cBq)UOS?NF6=rMhhy57eXnR(Bp{-LL>kE5Z z9PbT=1X|HPO!}#w|fCZ5Vr+JmNjIs|MTD$J$bu`Khb=+zH1Db(!1$*Kr#aY~5 ziqq6;J`QRuU#zYRY@yF|534f{L?vefrrOk9LpJ-^lS*NP_nr&wFxyC`w3}UC*6M!xov}CtluPbUG?W_H2aRXV_)VeHDtQzp#JRGoB*!SO|Z1imdM|ts>8yF!(Ji6 zo^}d)z=y|FvuZNJPiLixS<7f8nn0_ekacZ&#hj)IYmD66Hx8D2*OA8i-4^fM7;2)d z?_iq6VzVIMISRBM*;_;4TKVE(rLlys&iqq%5@3d(NF;@#PH=c4ix+yH4EAAV)CjeA zL3HA!jm*nBI7$0f%(u69;s~&sv0iMK`9WMzwW{-Q54Zq|j1SbKY)h^D2LeV3V6;h- z8K(CrSU}$Qz10T70vQLqxd7Q!zrP8ii#kCxedJe@3L#7v*c{3SMcEQFKnN3(%z%sE z73cA6)8DenAqJvi*v<~*{JR5*7^)WNtjmDk(QTl`G7wmBEqA%{7xZBkjw215ePX?lv?-b?wx%jBCT z8c;r=jxx8t_lA&pP?~LTDo1HHdq6rWQbG_${2~|mRG7zCmKZsYO2WOCxKVyWkI!Mt zbQ&suGIqh$5;?@p4F|oX|A?EuC~oGy9JK!@Zt5WypO-;&n<5dGFMHQ%q-o7nH3!K% zuMUk;@$s4Lpj=K{lfy$Y1^zDdON7I zY!7vw>(E6cBqACMgg1h>!EeXDl!Bfrnt~JiVM+1>>J=!0P6MmFjQDr0XnL8@vw^iH z)Jd`gmhf*6DHiQ`rr|HVgtJRIHLiy|5VUIFez@-PX&U*?*27m@aTH0uWJAYhyDvkI zDYCRVTmk|m_PzZu0~Eyn&f_2ur}0|fy2H$}R+aoek&;ibP$$Gv8|Uuc@USp$XkqrW z6^xd1twIx3wmH<W_rs}H(m>!&=fA<==jY4YaR6;lEeS-71{BMEbo-AAE&Dh==748$`@WnBiiA(Q`Ykm=BjJ6$l5md*N@e;|B#J zt&tvuw|C#}f4LeLcK&6m1u8|P4Mer}!ksX|<`eNHPbo7~UptpoW1=7<0zgNCcy2GhqOb7QdT8fZ)ela~RYMGpps;bQ#!ckK@q0T0Q4p zrT+i2^X{p&G_Wa{h}QCG8pQEWf$AewvtQ5>p9PA-%M?%Z)<(&sAaC_=(E2{>7P&K7 zO4S&R`-CAZ^;dQUeDA8N^IzGZ1m~9L?_rg6h=z8LH*Z#s%YH8GnXgiF;UBHpfV_3W z>w9Qr%Dq+`P86CzEF)a?D52H-I(Qz8k1xH`**W8m|D5rvBuw4Q{xWmJ(3LR~4Mv)x zQLY=h1&$p_Qoz${NDgD7qCCqn4KZ?1k4T;AVPS+!zIwULNZQ9n3 zyeIrEOE|_)>I&MG2&lg`sPDo-X^pTxVFt!ZXF&eKZdvz}2+q*Eb~J9qSKZh)`|wxL zLNRakeC1ldVy}=X-Nq5rJ-s(gQ!tR}g;)frUb^}uUPE^9V%{~xc2N-&zi+;Fn5|Zy z@Fl<9R!*$!G~b5<9%rGMsLmpClPf|Mt+bRq)ql!oXksC@rAIrGhE$+O4Td?isSy;b z4zYTXeSl7D<`eueas;|rY9Jxn!vLATm5YNHboQ2<;bNaia!0B+AjQ5rI@E0}q|VRvrv2Ia%0+cbng@3G9i9sb4|;xvf>reC zl|Njft%?qSiY~uFc9-|1u=mOj zGobxaf7Ee_Zs9QUy?=aCm3(z?^oUnBY>WZ@nazLs7e1Ou_z`!r#E>^(0Yq>XgFrA7 zS5{CD#y>E+quu=NrYqE4LlO^pAWJgSpvfbN&4=o*?Z$rd?3c9$&v}y|HYy)JmLqc< z%*PBh*O2d^w;c>0>Cksi4%SPsL`EER0aA$|sUH8_9}a5l6JyJ)Ul40_VPq_2@s`Fi zz&J%BUV%n5{^3hqfjr^P(q|lV3mvD)29v|*g8_ZDptV@=U=7D8^Sc0-ot+!$e}s%t=p{cJ@mhv6n*9W zuNS;N3FVN(dR=}8b0q|!nn$BKgupy4rZ`#|{wH!jnQx?#{Q=Nn%~6YcS*GK&n|srF z&W^vV$cMUZCnL4ffYN|Xc#HbDyiDT-{}*rX9nN+5{tcIum5^D4Qf5ZkTef5+TZqbD*)oz;MyRaJ zB73FmO-5vd$W~;Ny*`EKygust``-8Q{P8@`eH_m}9fyq1`+Z&Ke4VfJJZn8j#^coe z>f_aK$Ebk?l0z0Kxwn5>_F#n7OZ>SOYkxT61vuhT&yB3(3=)Q;9@A%_sBW>{5tBIn z^EdQ+DT6fM`S_Sv`c^0zsn0Z6Wp*eH_t4iCxWXsFpob;Y%)Mos5n7Aow_SMYnFLDy z9DE(ku~4xby~dNvBlYcHgf?yxLk;2ubhMzkmAyF+P)lrS-7&$bE4q+e?mci#X)kr2 zeSx6aA5KuVv@rhGYSyujI<%aKgeI8Tu@pkN=4W!?hcnq+bzT z$ofQ05jN2Ps+SC5dH zqZy#{)2c6r4QBIp&w)YMLZE`ah#_wMG9V)@Xok0`;HZ1bb)6j5GzYp&=mJKQ88f*d z#t-UQ0G~^dNBh_uTFk@!j^J5&ps`pGEm%SdiP+Upr$W5l2p!-fvZK*p-=ma7KSDt{ za!1Ur6DUxMpR&3n;)5^cLGVb}jUV&*V-!$FwY!BDeZT@;hs8V>Y`6CP*krkM@(eYy znidma=tD2YEVT+KjM{Swr-l_D4dFXj@!&LE%i@eYzL)gBH^wPt;>%`1n^zRX+LjYd z!zMt*a~v6cSf&d04m*ON3Rd_%^@2t={Xx1*`+cvH6t1SHrYZp^-~!azaNZF4yae}~ z(U$OaY)R2Yc8zYIxIIza0FjI01np0VCaKT2zhnukbtD@fz2bk2t>)$22U%}Z30|K+ z?zv@(u3dz4=i)V?YbBNx(311GytIyFaEXSuj}zU5acDT_sR&>!QT%J&7$uxbWZ(D9 zuJwEEp9Ba}@Dkq4EpX%R{q74Q|NQua64YmJ``e>zOZ$}C86&+2U-pN714O-CW z9a$g$&V4%^SomgWcETn9f(imAEWX5YYi59yfEAaqZJ@|Hy7nxNs;`48H}p+OZGU}z z`O~FCZc!c+|Ab@TdY<3MZc?!Wm}cKY7t3 z3USCNGnkZ;#pY23)GIji>cSc^q0gI_AD%PfJ|u8*cqk+wyMCC(9<|FQ-C+yu;X|9bS%XzDz$2zz z#m>!o2(LPUUe)o)``@7lG_eU9tG{R+DGOAinh3tg?=V2ssU z)W=bI=QjF>TW|wLcM0nGd#{L~{_K&59qUQ;bmpb+wNJkDhlh1lzkTLE_Ut_A5%^Wp z2TeLQXGx(P1mKYetOJ<@-}P7k)y8424G#oG-+#72eO=Gta>;4v<79^L<0VukV0eVi zSDQCH3Nm$QegX+<7sQkQNlN#3&;?mr&H-_t_863kiVi0$XJK!|G4NJDF*Z8}u4TM0 zF5|7ook%7)kH9%|-Y?560ZgOi^g{4AG;wmC$~kQxHUxZ(EPC=H&4o^*|I1={-WL}p$!60E{VGYCix?#8^G!QsKI`2vAW31EX25?Cmc3DnmS7 zv?oU^47vOFyJz`P#$#7DNyVS(m2A6i4xrAx^%;%BYQrLrHVgb5p;eD#>1S)?=jFsR zfskPmHfZR^9|?wjNw$kX{APw&@`mm(EDyR~?d5@dgHRV<$OLe9!hwyT`fg?q0z@_e zm}V3QqrG(sT3{yt%l=NMFRq7`)EM{&b{HFYJZ+2QJ~#uZfd^Ue^Mp6;PxZ@ z%G#b`?dP-**XJasr@8%UjJ0*xtb!j(%g?Z+Qt( z0t(B=kDC!#ECFhzv*}T=v;0VV(P&l;>49$z67~R7zlC|U1$?*MpZtzJMwh1w z7NU1bNq-tfrNR{MYdBR{p=cJJP)3|@8*E{!y%{j!^uI= z+Vu%IZyB9^|C$7>EM=jS`s_s0v!2xjiYb#f?q4%Bt{NART3`6SnFu3tuR?h|mx6@V zjGl?=X!GSfw|2(zls^)*~Irx^P;+Zr#?rk zk&)gadsY{RT#HIY!MBdTB|R~x;)9YQDJ^m>P3_O@vC~vbyRI&fANXmQ>{z#SB?~bO z45-|$#`yaWin#sCsjE%=0wX&pGzc#jjD3aPCuEWUsgdJI5BpQIzqOArS4__9mD>w% z0%#c;6lsP)jx*>+sS11c+aIZ?D@Su(hJaTtf+8;`(i}*fnwwc@$|{RmV#xMY;HU#| z!rYq3e4;X#xu2~~>1!enE;j^TT6`amvl89v03UujoPq0zZ;fe~8!-J^D;W*-#6FId zSWH7%Kk9`qQLicS@VAgi!|}0+J+tvw_ut5YB{sRRc}WW-<7HOVbm@O;IR^2k?z$fz$;$m#5C1xdM$T(c`{CnYr-CR~zm2}w#c-4TO!mnRoU)>I%zDHTYepGybq;*5%lUjVx zV_IjLN(2Lr7Si5d6g%)_9lYj52cE0|2*x%x!^7KASR9!)w?X2dCjn@S41egc3lb{t zI6~Sm^>I+@Xo=g6Y;6@(gL~?F?o8{qH!^syB)mO!U$+d(z-(fu+j5j7ER0kL+SIg@ z6zo6pZr3G=_NT$eo4>`)HCT0R{pfrdggntO+xgU{cl++w^hMIC^PXxn=gu$aHIcuo z$*XZ;xqegV;s;cm*=>eILky!k&}^P8Y7KS~p(&DomGP;Uv<@)d2^!9{fpYfKV`M z7p=CI!12DZue!2@wMWr5Li z{R3r-Dp&xcMDOYDfm>Gx4i-#8Fh_Vmvq6+|7|Pp`V3a+s;f%7zLSr4xUEVZtdt!ys#5)eZB?G7Jtf{$l>>p@NxN; z6hjlP0p~}b=MV5<-v-rR?myULsTUAtyga{b)Axc0FUFypLUpj~3G|6`=zOAPzoFRD zlq}f5hB@@(1g_W=X&4V)@Cz{0$34wRsLm`OXE{X)&Wmj5JE1GS=)2`$xvv0AcMYFG zt;1eW^gO)t=5ia1U3QSxb^0|dych`t64r6jr;#^qk*n_1?*rU_rG`j<(@0kS3d)Npj2hyN42>;R-Q;> zjBlX0h+3{t&3|(9Rh(V95?R|H(7~XYqeKnQsbW(=>J*P@bo?5rI8y%5>uD&^vbj7y|0Ij}wGjec_PTNQk%_PQtVvBl=Bh zsFPa}w1e@E5Kl4*cvRQZ8*rf%R*N*liH=SY<6K{_aL_|4je%Q8TC7 zHzC?~k%0;^FLT6y0HL1)?^Kpq(=%~!`c!n+Gam!`!-gl2Ww|zu+WmkR!)K27RwnS? zyDcx6bI5kTN~nK%|AQOL7pEI8)wSHXIH!)pzk9TC`0O9KV`tO{a4Fklk29*tsvk3a za8`OrO<%Mcp0Evxq6lj?aKyn1FO_dWfJ*%^DFwn_MJ$TNCHrmM3GN?u&JKeQj7{i}U`Jp0b z|5Al*Pv;AQsdde zBu{0Xn=V@Au$PA@Wjvy$jNNWrZu^|w2_3Zwf+<7iF1gvKOraSPw?$wI4N={bqCS1= zuWpkZT&3S1N{3&-p_|lv#5|tn%&Y0blkSxSFb_p2npqQ_pBEEGn}*iB5*>~S;@&G? z{z2kf%7g2*fzkgAyE?#Dt zVq)CLHr~zp(sdcy)C*zBaHvQgf0*H3?CkCp7fHv4taE-?L_W8#U9x? z_IlpCXJcM&P-#k_g^u|T=EHv2{=Su4W};fHUmaZ+uOcvTUL2OSUU==UVN&_@7k|Eo zO3-tC2r8mXWfNvJp(bdF`SL{K#$d4*Gm*DN(L>?J%kseG*c( z@BUC&Z-$&po69G9@@ysy!?U~bC9~A_LlW9wd6q_N7w6-4RrAt0jM%&iBnR$qB!zv< zU}T|T<6V|ScT*7v208kk6}>igx|m?&oxXp`VDWimH~!S^)C&>*wHB}wv5UFyuaMz- zo`W^fSlowR`>~zw=<0~-Ka<>Hr)__t^}>}c=}neIO!W~@luQc~ z@a6b;7h!JFtb!y)qcM%)&v!h{<3pnh6lcRrJ&cVW!>;Lc2PR*r>{+|Vc-F<^n=~l$ zT@q1AVY7SR#4`nB!aW+(R>IYtnfb_Nclw(pqQ-5FCwpM&kp_(ltNgO%-Jp_tjd4+} zqQa&%r7iSz*>}@)YB0S4f=-e4w9S8J)qG>e<<(Qz=p%wQw;_8na_YIO(3aU-Kx!Xe z4#Uedorn2&<mAVWAW~-w8`E zNtD`V-<=@G3XHi|;wCIsJ>>U`00icKah1fqjg2QJs`&=BJ7}l)YrVtY7}~E6*r~(v z;?>?4RmZ0yKFcP3Lz_(tK`W#4?W`m~EH+@RcWdBY9SvFPD>muVWW^bYTrgmIJ%1mQ z-u~?MauH8zgJBgWw|)#-^^EqJuHS$)rs>0~+RW(wU4xg~E;Z9>x;`5G zVDI~TNb}M8IhDoG7|r=bZAad{FyB9Bu|kffRXx3i41!r^+Z(I1<@0&USSVA=soDr6 zh*%OPPg(!cw7%mk*7#+;m#I*;f7W~HzH#nN_C7gxb6|W>&YR9)3PxYwC|DVzP98>U zciF40Hf^uMY+f19p19E)1@^=7q02o}arQ<9J~;tA{g&wo*N>AJi}TaA1>TzYtS}lF zu2pHvKAPCqT4+-DkV3F};&9$L#o}qz_G=qDI;P-4hi;?;avk>t8TY`?)`__ZtH&(Y0o5#BRleIaOgN174klUArwhGFwbB zPTfi@sX!1DZ~WXd`0D+aC)(UG1oqn&eP$IpM^VtR&PKdL&aLuk_k~5P|w-4+On}Q+t;#dW4jc*TvI_>=rcBE2BNvL?1iC z%FGhD%|2mf{&1?26$g?krC7oo$g7gubtLQp!GdmbqD3N&#N%OJKbcZot(jxU568oD zi!#$O*u(i2YOy7zH$M>B`>e(F@P9MZ9Q69oE|7idG`p?3facOM*UAlc9LcmgY6ml~ z!=vbkoSa9!W#@mh9_NTwL=Dpe=cF^o`qK1rU#rxht2YXXo9o++5!(7HMR?2z+msM!RRBy>>h-9NzVft*#<5PB3&d%(G<@ zV&VAl3y&7Bao->Ke0@EgD&1lYC`F%S% zH;(KpqHSY4^F5!}@rbY;Gpv%tOlBLzhMld1Sa+pc!jXJ^-gw{BvTXDkyfm)O37CWE zS@HEIgS~W6+!EC6oh{?s=d~)yG9xWvX=MPTDf1FTeMkelutBUOUj;x)3XLGQG7zAjkE z7TKAEvG8db9b4NW1^3Oc_%@=7`oMi}*?td!J=%<4;-xnH3&6?SeNHJ9aAP)bF?fx= z&7tv7HFTSc=r(mad^{*KPE5`Cwz(>AUPb_PgEXu8lQXU}G*@ddU#yNgjc(+*rNR{8 z(%kZ@y^VY|N!X;T{9AtJTq3WoORtvle3p5#{XFbrcWIo<&MRjJ;)f6AeC*ABkis5c zSPV}s5Vl_v6us5>iF+f>)n9nBdEOk9nGTfs##TN|unT1~lJ5Vt-pK89=?t7NWrg7H zda2!5N}E?kU-n~Wr)CX);5uRCml|+S!}@H#^Y0HMbA()rDBSRfqs!MOoJCxx@r)w7 zUyT@=J3nD~cj&;SbX8oZCb26ojhj?<-6yU)4#IYcPW`(0yE3#zCr;ZJ zve!H>o=uBO1O7}+U)oKBtS;PO>t7RbfiXVQCL zm97|8zA+vyfqM!$#oLYcVUKfJ#@QAt)$H#sDSuvUyD@fqf4f;SQB{27>*FU`E=TT* z)r37}DZDa;+q!>0w7yZ%lo<#2dKQYgn)4qb@A97gdtHA{@wNHMtm-||Hp`JO^w^uE z1qWK?=o+8#WiiCh792SnF6e?apG$bP`TN9gY~MsaNJ{K!z&T+64Nx z$W4d3qm;kpjdj-pd*h&IH>wZzNd=KPjqdMlQDZ#k7kgSZiZCZTp`PpvY)RKuKH{)m zyCjE|+FtL+&1c*UG4<5d>VL{pJN229_Ww*P9JSULwx~RGnZ5&-ZBHv#c(120sH7*j z>VqnT__TXo9x%w?zjWXBMZigdsoQTTBmAUnA%Gg>>Vy5#SPxpx1p$^7tr$) zTjk%XxOQQAJ=+;!`MjNWd!LsAFj2R=+>=+2VCqKLNnW6L)xR6vXuWA@e5Nh1#7!Kg z3fk6O*|D|1Z-7Cu!T1lNBo6Hynl)k_N?JFQAF#Imb`Wjhq-IEHA;dP(`B^bFKVK+! zLvZjb$OZaqjDkY_T+?Z6Gd;g1-+d^&B2WePxx{%8Xx)-GeRd{OCtdFvt;K$Oo3y)N zo_ceCCuv_85YyL%hW2TgW1FX~b2PTS8pNEAe~@66-$k~9%c71~*lAi0IH-ACVVT7T4d%?N}dm z0*6-42gKRptgTdTZ#Fr`w_eC?^g!F$&1$Q!Uh5k`0P;@VfEHEukU{d@gpow60Cfym zn`Mbw=}b?HiE2p9dg4}Py8BH1G++nequWh_+Bw!PXE9|+7^7T=;P*8{VtXvQ^9K}J zMaSlS+f)s1XP$=`+KTwjS#>tX7|q#Rf!J6FYzQx0UZRGmMr18S*#P}sg|tP%bLPzx z08dW(h0yp^y|VoBP#8AzsQ@l*nKs8?;xYDnRHsxJ7JAb^gz@cdED|e$tESYC^};ml z2NNXz{2)nJ^TqUlk`qofkZxw@7VGQ7x=_duCTS}6i|V?Wl`o0w zba_xcNDj$e%f=Iix_1QlQ$MpC@1JT5{388f_gU5Th1)#S5Qn-T(=f#DSsDK8!I2Xg z{etq;K^l*nV)rH__n5pF%owu!&l^FhVNeu=FV1t;w6TNpBK}lJlgRAbm))?BU28?I z(u)T}-OMXjvkN8d8zK##jW*vqgAPSR&9~2d%VBw0wSi&y@QqoI^|^^EmHw4e_+P0= z+cFnTIsX~6c@T@6EUphla3C>lIGY50uFjb^}sGzmRp3k&{fF+*CDrOVk z>KV%;^=zjU4G2EX!=6-pPKPDH{BjN!Vm-H>oQbN&^yWvMAhZAI30q#Se%`p7Wp7e5 zt73l@+WNe!+H)DC4huHnz0=UjWI8x`=SRI)uqRK1Lm#q;W0iCQn=kgHKX%YZ%3#6z z9g}kQIq|V;Qamii#DkwrI$*?YY||P}srH?U-X~O(8T_WIk`INul3*Iz5S8@nUJ1uA zS0ECo0oBDKNaGIkwHWoPmeMGA`PsYRI93zrE=|Xng(`J%8%Q*M#?#Te6S=Cz*@=QQ zwXwa;66N1=zISkaPOD(}X$kI3-`@=zunfK!Da*s6ocrb?E<3DjXBWSSA{`{>p3Gp8 zc&%8nPSV)(q|Z61aHGHj3RmgMTxD;0#J0dL7}GRP!^~&faGQ~&bu-n!DiOT;1%#F_9 zT)4wyE>;&QpH;OW6a4=6g&6WXfqOmvBoab$Nmf;CMzS4S!ah*;R5bte)QTCzDzIFevQJ%Ko#v0{nFv=D7!`ZwCM)~Tf2Ucvgz$oVi z4M;3ujdC448m8+B6=!^^iE0Fk{GFXo3KB0mz(E$m{w`fDx7f>ft`3@oi4&epz@N)0 zXgcjua%MhQ^{5l3m&Gv0fW=?f-s7`Dx0cRLEiq@pc~>JLJ(1cmb1Xblca%CpTab3U zD_`&2=RYBP&254j#)PafJ=Cr&7rsP5I-+?^;M~7x)d(`pwY1Rwcqt=);Vva%D9?r- zy%(hKaeN9X|7_KgxBZXys|kcZDa217b;=s6fPD23CD)V!yec_3-)=$ELz@b8pV;FPQjPYg>jhSkNR zl%VPdvArUK-`WRlVcCx*IGC$|xLF3#23}ggWM${+JHgY6FINGiXBhjVx%LyhO*g`qD>eO!XBOK)kw$y7E0?x6EhP` zR3G`FplT()IMe%COb^A&XM615n@W=o`wbq*-&fc7SPj5%LvXiOawYw$Iy&1PHoazS z_N+=&x;ImpF>&W6*vEO!8+z4Z$DA-DPpBU15l>MVY(1v6Q-uCBpR?yrc5?8=!rt2$ zY(x%K1YR(tp3elhy(AOCrC=TfM}j{y2bS9G@3DQKl`jF+3y7;3HIxuYEu4q~+L07r zF?o6YJCRyf=dId7J+HS+iGbSLMEgWvY1pwj6<;t?4Z(3iv}=p1j=cM@b}FyGs_Ut6f#aix z<5Tw>2N8Z6D*8*;aN*N5{5@=v_PF`pk&3fjwf5hewrd3o2#eX&cTI9kRO7;35mw`u zhtSLU2653VSTL${CgCZvL;4h86VJu8J8>hM$lPWIFYRs7Rj)@fMd2GjW{M5B+-iQ0 zNGqmmC_=^l{7o42X$f;O8)d%k+6czHjFk9c+9-0WO?bKX2vJ|7nc#2J+VRZ|i7=t@ zrvN5XM^@WymSXsoZ@)_F+K<^`o>M*y#-aY#Tin=_5Yq>>$RnGL9ErGE?*?S^o?tx0t!%TfdN^Nb9OC_0D0E&g$cqe61LR|tiD5^vk*gi@qIN^aU*i;$StqtBYUVf7 zY5Pa0I`&pZXgYmp*>3sbW-)|N{|P{P$M;tAmDgY>`z;J4b+>!i zb9&zmR#FqJWa7od1{sP@eSp232(hMv%PHvfq$PM_7?FeP|$V z0{2~0m>2m<49{tFv$!M`QilDkj9+*X%YN&xI=#{mPy*SbRe(kE5fA8ave_(5DgKR6 zRuulFn0oWp#RL8zh7G-UCDz}Nq@n;eb<(J#)Ne$2(I)ZpvlW}-;OiU42YA_DN#IFz z!;t!rH1)Z*ObpJ=lhnnF6(?UNXOuD#!Zq|g4q3zG0krt@78gA7MaTdurY^;3?%nr- zevj*qoK#_ohcjd=QNCI+lHVU3VVCzgRa9rJ0}vDz3sX%vK2-_1rke5IYIN`uE-wi{ zqAc6_zvbwDo2y6aNVAhsItJ)@Bn&gpgPd1wH`c}S@>T%&9eyQd!t*gN1MjNi&@sL$o zm#^JWu#Zo;fYN^m?e1Q^K7vQS>moSftjSvuTBefj&*At!2aZ%vP|_W| z19LBCVL=HO=eIaybT~#2@xyY&LRg^U87FACU%ge8@8DGJ^%0sc--7-Xd1+Zn*kv`f z_pZ(w?{66Q1W+64R%fN|031CrOP;hX+bFR$?4%a-++t2ea(7m(2mZLxic5&QrgwZ0 z@#%}Q87!mKS`z%^ZJ9p52jbwxoxu%8Go+cX*_mvGh>Hu=;BpzQ! zMonD@0Ad%*ds}i?oBJzP-4DQ39zL8r$;WA#TI<*l3&Zx44O!7biyv1A!06^e0n-aJ zPFxRGaO_UU=Wu2_Tji`9|u4hr-^s+^*xOK)T1*c$*1De@Gpt;TlBj=bp9`z!Nw@ zbe;}A^h2^KOc6I31oOqwSM;9rx5S_2kEu6TPqyvUk@$q07ACFECI@p}1w>32Zhi`A zjr-0pgI9Z@hc#vmOm8|A3^{V4hpEsQ*JWHqdr=j33eQ4|bWVR2phN$GqzjD;?;P)} zZe`Do=-aSrB3>|3GT#!j3o$RTjmW7vh!xgL#N!i?o0>!3t6F8WIuk|oEe1sF-0;7T z6>2w{mkWzx_aN$FF|K(eb`%UM3RVM5I+()5UZ_GoTCXUrkp^nX1Po)*y9SAS>*nLTEqNR+3mV-2`EXQWMD&wfn1>#SQQV00rm9cr(U0r93C!#X*5 z>T-6-ue6Nt7-Sfi>f+_=o?psJ^nvP9I;{FAHhl?Z87M7Dp<=6_f-r^e-ytLpUnEkk zecKKjiDglzlO_cYAr@p?_?RGF zJ#}H|_zQT8YJFaKWS@;UcTmn3M&h&VpUPkulC=r}YsuFot-+mq-jze{VngcjCnUDN zuW#i0R+S3Peb5AO5!P!b{8{2?>R~YyyX(TkV&v`)E0n2g>~?rD87YR1Y(~tdTnr)B z4bIlbcr}ZTK8Fgiz19MGih|ZwfR4@b*3sSAq8snaRQJsUhEm5;Pqxjv5eZ59VZi8Z z*m_|Rd3ckVEO<~Ik27A!{XiV50YXbFErYzcyae*$$?VufNoY#%fr5~Bfw$2^;V|({ zU3~~?I9loL#exJGig4NoSJ}@O z8gT-En4cYFGK8uXN#te{SgD{fcTr8TO3EuqrFCe2DD|l42B3O*nKEU5BNRPS_CK4p zr3CX^pUE7!M@N&vNF?Ls2<^;U-CX~UEhmXjx?M(^YH<7xnsr$SKUmsSugjnc z15MR!*#4K7AvdhJ*Cbf2f&M!FGZX?_LPr8++%%-#0i!S>w+Gm)I_&$OJ}KUM8+%X% z;3XG)KDPW=@GXr`wWk%7bw1|WLl(m(BGQ^(O28mE5tqPKy**yvK84?ZLk)sE(U!tEiv7~H%75Fx~_Ui0QQ2;&X}Ely-nO?|F-MrF;}KHb^D{wR_g4{_-)h~!J1T=# z*|}He^c@NT0FqZlS}E`t=~4%H+blt~QXygw0{6PG>k#Z5vN8`e^`9 zRG8kZ0`2&J7Z}3LWJrgc)I4)2P>>KdUpu;333+aOtKKYfZXi>5Tlzy!e3C6@ElseB zZN%XtF{PFeL|xq|7YK4O<7^qUCsEc?1J_mrV<*kgZ)guXxO{V}hRAo2Ta`S=Cpn%K9 z2&NyBU1l5g;)fJIXrqW~xHd2Kvf=zN4MX0h_R!Q(Y*b`9+7X4dT?SygHv1~#%(Jn! zE3McJ0?P-r?hsjqD0EhPP@wu;_9Q3P&t8NKAi!O8;hoG`+3S1R!Qt&}>G9XAt=e{= zIK)-8XltKM6M2FDSeQwRE;q3aOAPr@6kw7~9%0y)!VFJK=zwFcUU_DJ4MBkVQ1apE z6RdE6996_N!nZxNbESU0?Bsb^|ri??izFCBDVV>ty zl!2X+<41A3#iEp?~{w4kEU-jVt7J(4Zoy=h;17c&t%da&^f7b8+i zy>mP7`U@8z+wTmj?FA6 zBXm7#=v&BPUQ-EWuAkYHkZ-p@U6l%I6w1NWyK=gjo|tIgo#^#~FmIc$Pemq&8kIN3 z<-C%1+RVG5<<4kNOJuRIryFP{T|q9rME~+FH3&#)o}vCQ0Pu*I_*G(f-jphT?4((o z0}`p?q^%&1JjE{K;RPT`=w9Q5SB21Ko{n(Y6n*<= z{C$dWtIC5wZPLWGIp;-o<@wB)kWM^$_t&OLrO!|Oju?B_rpd#??$m$lNou~&MmFUJ zJC`i!1$mD^k=zd*BD089QFpFhz>&PtOCS7a52Tm8twjSSs&w;EDB(OFNk3YY=DzI- z5U$!vS~?1G0~WItB!fzf7on&+3mn0GAz6;P_1}D_ebUlnJV~BbEb`$sA>b^Ly5Tb zMDnQU7@RHd-Sqi<`n6Xt&c%~+$AZ-qSFqKrI=9Ca5A#1V98_&epqP{npi%jk{8P5% zL-KJANi>-moNM9!wE~Xp>ISo8L43k(aVIOv`2f4d5v2}hRZ3y_Ftsa<0AB}V>LZHZHGSvY#`%iJ|Fid{rn=X&IlHD=Wdl$wqq)_7j+f$_&(2ukqTTT7yr?_wJ(?ln#3t7I+ERUT@Ndtd4EvQ;}d)DGbBBjV7D~XkQ-y(%A3JI^#0J6ZK9|gJ=91L}|@DG6ZI_@4aTTh^g zyaV>!#yuWt*N)$TSQM?Zj7;0D+wlO{WO=!?V^gbu>eg6e?OZyJc+Dcqkw*MUL(kdN zSyWG|P^scxx9)q{GV1yN7$OG{Fqxf9Rv0`5A=BR1O#SxWmA*80W%c7_^h_lS(Dhyh zT$l?mCievvlCbxhpk*7S?`2dPGx?S`bAA)toa>TvKs>#x|g9u8rY3&?YeED)c5fRhG4Z55<=@54q*&kBf?#jFyyO^)46WgCuEGx>c|ZKK0Ce~0lkKEUhTR#3*`&SM zr1H7dES)NL$Te!%6PTn=A$}sKpeU*hQI9Twxe$D|1km}GQ?nr?cHVcVIH4lS??=k+h&38s)mR#x4xm;=PV$r_Om=> z$$@c|$YA#9+FJ#k{pvPwCmI{0_@5BTa1I5xou|In#529Im!?C%f!=Rr&?*?Vmx9$TYPq2nxy z@ABpA8e?~<&_tB=ge5N)6txBH)k05X=pg5>F}wFI%m}z;8x$MN>YW9?w;D8)d6z?N zfT$u5DmSzD$9@|H@hEX<)?gmZM_w}WImL}1GNz4Ni*n{jj5#da+gKl2@)1OomJQeZ zioHOVL`CFn2@fX}7(EaIvJ4;W>IB}=`tMNl?tEeTs?lHB`voPV9>Q@Zsz&-(g`a3h z@tD&X+)Oy2nfYBq9FF8u6z7_8R$jLQqyKtH&dDUlp38QF3foN5Tq7q=-wEWZ7bhzv z40b2d8ZLasqpE9D)Ta%}XE-)+B(c;o7_6>5X#l8xH#BY~ ziD*ofUZUq(VQD?dbiDDCl7UTu+>cHUwodp3V0S}Za zKXKIybv=nHV*Bh}ztW~~ZDl`!tmD*DA=Eps`z}E^c5`Ow8EGG?i%TcjvtU%44dj1s zX-n;!*;lRI6R3-?ER48!A_eL@JjLZAD4>OhdPNGMEFyCq#yLgWe>cvS&@b6kqYps& zDA;uKYyPEF4el__Qgr3o-J1tFO-@P7DwK9j+=0U;Dr9Sn0xVu3+iYfd85%y2MgP9! zxgJG$ndjk4*h`U0eX!Sjl|lH;Vy-;xsA`3%N?s&2qP8lL$QMbPh_x|gANS=LR0X#}Em5y;8T@88)c;;9*j=svrcdYB;Z&_Myw853A(i^C} z`YXYHa3!an-n~J9gWGg=84(n8D#s};VcjvMLfKqab~kV-bP76f{Bnp&FC}hJU9a}} z8~x@-IT)djus9Eyzr6I$PBkK=u}IpczeSfL0GIQzZ+wm;Wh6n8Ke4yr^o`t=@I51J~DsDnDy%$dLMQAa}4|P zQr0*a27|TJQMpH*9;c@E@`fQvwc)>mSmAJb$E3joGRA6Lz$axThu%>+XqU7@oS#^p zThc+#9TVNh1J@yxdkYL;5+p}=40JjWn_dk8`Us7jq>jS>DGP!f&lh33X?P&miGqIQ z*zn>dBvZC0+417Aeh0CFp2Nq=0C-`hNbB=78-%CfS^gZlsV`9=+r|CId|2w@{p?LC z+X_IoLuu8z{>ru&l;nO^>_#LW0*L3*xejGxOm~+tpY9rTPzZQ~zA2%imOrPr3S$Mh z!T+z=Z8NtYR;&t{LFM`LZw=`4nCt56i?6YQ$~S9k-aSxx6GjNJk0qbddU(k1vLCAR z%p#iQKx~0Iq~g2lfofjoP$P+%{fq&QFtR~-&)z=u{u`BXw|_@!#rWkBvO!Xs19hM_Ca72YaB6G^_ZXB4ZMq)AWati1ObD z;ezwn@F^rVuf+kxD4vdLtislI-uGW^#ZnN?e#d-Vdj5fF>(C*t2qoDoH$WS4ogU{l z_C}*KYvOpl?^yZJzpTcj!=yM{pZGC243okXeWZr)1HuUVxRt_o`7n+dp%~OTW;0Sk zuabx@O9kP*Ip+)sBg0;J6t}SvyQ*Plqe}vtFCdj{n4@u>NpTKe{e?!s>GtHTp z#s4YA__;3r&gZDqZOwQn%2gG1$WEQ#XBoW>-ZZI>tw29dr$RO6PO; z9ZQFd7qQaM57M9OLEO^^nZnn>xmZDv{>q}5)xRHpgeiQ|qys^c$Yj0PfC;)iR0cL0 z`T~E!_)1m>;H{-^yHyDxjue6`8s(C39~&{sx3T#iSx$?yi%ssS#mYJE`*Wlu_@j`d zr;az^+XICX4!8xC+5Z&1Itk+RA1`E&KjXVDdhq`3L^HlNqgMUJ6m6%#YmbSc>@~9} z0*k1F4`oHS4ThrTtpbC0;ZJcCp+cPe-%ki~p9=X^apCv{ZVx4WVC!G~d>8+CMJ;0Z zt^>h2ng+#J*w&n=_Jf1;ArgZ^?<}IaQGUdKdLd}Iqjc|+Z4Vo240#OY*yb;MML|&n zGrku;@*LDTPs3B&9O7#*2J_-Lk?lDJ)wj!IvV@9QpMZZnODf7=_*OMkF;UZf#I{*O zJdJSk!O`GXvXa_+7*@Kk1as_(0Lf)O)jg(1_5lD*iJo(r>p&5S66!N_YA=L2|7ir| z%n^OB4rFteBE#p1~0do(p~n6NtK2H=|Ww{Kk^z=A+0Vajb}9K+AR zl_Pdw6tDw>bCa*(?g1TeA{r}f@B=e1rTT!ZuzXvrK|Ox_a4XE}r~Z#H`cjroC*JRd zfh<(}!>-3G|yBIoy3O zoUWr{@6&<3!hNg=2T96boE3WV%f8S*sD})8y09@v#;Jc1oYMdn7Lerz@2zwIM^jvdA1~ z^8&F&%HCYS+NLiue)#sl#yLZ24zoN>u@#~|y-(>Nj$Xxa*O54Wf%7}=NtA8O1V;wx zJhqA8BNv$Ce&55-=DA~ffNO}00hFkR+hh7KGXq3_zAoP4^)|y^=@lah7<9?G`%&_b z%nth+_CQQ#aIw+~V=T13-iI=?8UX!9jUq(2_aK z8k0E95*qyiRw`w}+STM=5W~(S<}x80EcF{q(O}7k>F*$abn(SE$kF@FloZJEwM>N5G8s>o{KoBqeCC0F=I}i zm8D)7c>7jFD9CF}T4kg63jMyto}F{OdJr4Nlg^aa0+5z8K6-73g8Q5vB;=oUUUini zNyZDl9!Doxi^5A?|18xZ9N+{a@kSj?h2DmuSVQ9{21?%MG`~^2@KCsF!&Lw$7ko`& z@*RT%H}XIV@8~LNTi{%u=Yz9-s5!+F_Ab&VABXY9@DsYC%67(hNvdG^3lSu~!Nr`HS!; zAT)#;eZWP&OQ9%M`2NRZ+B8rG^RI5&2U2_GmIyt{%z`P}^6gFd_oD);RlPc$F#S*fh7K1ZB~KApZvLns_} z(5fTUA>0Xa31NGw#-%K{_`FpSYy;a*DeyffPP6&TQM9#~q_Cd!tATiLn+^L!aCDgS z;ZWitZKNGBof+2G#C0}}c(BcD+5Hx#jogn(jf!wVFBnekhyPX{pJi_m2-?$D$8Mx> zt=Jn6|KTHJ{8Cv99lt~*a?LXDdo>MZntXN0I$)aP_LKS~{yyClPu|biBYOR&7DaRQ z{Dy`-l=}@eSBla$`V6=mtB9eylFw;Zqq2nZ8zWr9R>|wT@EbE}AU$?-BAc%XL8u|J zz%p7hE0Sdvx%GNwc3LEw2DeTRH5&JfZ?9QF9+I?>vzQy&#B=oJuKNKQPc>{pL;aJ z#abUW!KoM9n`1Eo8Zod4{@>CP@_^c&E_WtOY8uo_#C%o}`(YpuQMju#ZGnU^g6lnx zr)s5b-+`Tftid>%MxXMrk+w($`U}+<75+5csX$0e_LxerHKzAdR?nL-!kl%$NSG{(}gA0s8?*&VzM&aQee% zEufAg>7{uB;o84|3LcIo43TRaMVg1QO=Cv3%?`-79~nun#E&Q4%E%WvITI-r(xh9N z=LN*a`!4#^SkC5I3SCt4gAkX{_guW$X(N=m4sBYER!TkUkK($izr8jsxi5^?KU%_= zchtFOwZ~X|UW&17Z<*;|hHayQT;UEi>!!HpS(Ck?_d5J=f02c2q!x(I=E8gbpJ0ld z3*RzPB|A%>7}Dhg924jfK8}M-vitPzDpa+(e5c*;{E~RQC7D3jJLDY+R z#&uI{y|E7JuLNR4Pc=Bz`^BNxOe!JfoS-a}ME?tC{8|O#Jb@nz*tYlo`~oEabcl5s zhwNj}M()KNu(bR~MCC)Q<+_RwesLe?QkKUf`yk(QkJbh@9IwzG|(<}LW^vn z&A@y4xZ?HQ2sTamqpst*JJ1DWH7g|WH_H~`KsGZ%zBZ6l)sUzm31&yQuFE3Y5~kVr zPje`JPu;-*{(WiCtxeF1A!~b* zWXB-p z?;=F?w=?-80{(t9JNjrVyU2fZSM;n>YDW=C*clkB>+A88EC!JF^gMxD5bT$DyV^U2 z4VDi1UBWfJ9gjFY;!x}~v*{hlM{^l~gWogqQb^?Ad;EO@fJaPJ4QyP`eIbWWOym(I z`1=DUgDpoBd7>;1m7=`cxX7&*Qc{gfSK&EYl zp$(ft|Icj#YaA58IiU9nfiq(CFom4J*0taf|96XwJqQ54P{`c_TU)y!km?!?o#@9H z?@bzaBe30(t}YMzawerm5nCRS!#5~?gv*>zLSziJ&RJ{*^54>A4Hh8RV<1+F87CTCAZIDp&t0|7b&=g!{PY{!u;>%Q*dCQtQn%}i+Z-L`d2~A zn(C5!vuLZ%tG~+Gbk+fKiF5{NdYlmYzexM;cq-rU;YdPOMfOT%q{vA2k*$pEicn-^ zg=BLmN>*swJDbWTrR;2>guX^nDy6LOUN`3m_4|F^_w)Ya^El@`&vW0`eeLYPVUy62_Lxy5L*vVZ@;^kG8D?eDLW(6s$w#Zna*~-#33or2#;CJjUj+ zD+X)e(-Yxej8Xe&DmI_PRp^$Ju^3 z+>=(MBkTgFz&-u=x(I$@u}BzR>0K@-UF8jss~vpT33R|TNEh$!O|0l56rU*RMR7AC z5}GZg+v~2M`wBt%tJJoWlG5wY6^_vpd1=7&tx>fVN`Lq~Wy%tUBU3!M#uDJE zRQWMM7*t_!z^^wN_k&g*I8JpHamuyFNfN9y2!1J5DBGPrCH(?S?FXoxLFZWZqR{y! zW$0C6Jv-yFau(HlTAQAxp<*4>HRkkrw~xn+@1xA!+tZIWuX`w%NeAF%HctZEsks5W zc)A7f>e%Y?)-Ah+<{ETTa6WYp)uyLGT3J>enz-2xr>#-7W*wEP7dw@JNDR&agfNvYy(e4f{hyl+cWAR=^5W{s3#02GDDE zxkBE&wj^IUM0HEaz_+1q+WDZADaz}_1z}&;LD#InkVY5L6%Qb>R{ZOyVJF(oFNKst zUU0|WJBEU(uOO=`4zjt64i!!A-H0;$fHIy>?zB_h0fF!Er05Nf2QUC*uGc)rO}6fa zC?Y6asB}g|3Aq#eDVJW}LsoQ*bGypRm#H$1O2kK?H1vLd%gUBaG`BP?+1eZq*S5O| zqBsSrJ(8eQ!W+=}IBKS_>JwiDkpMlQ$-kY{05lY9S&L!xT3EJvm! zBIMU!ekjIgRdb9YI|Tg|gke%adjZ1+4r!7H%!)Wus_Q)d0(pes5p_2KfkPROomk7} zfNOQ1@vvQPv-U&=6-uJm8OJS6+vUfGOJN~big!8p6~xss$YHVLz?6Xqx5$lG#d&>K z$!}Hribq`9$n=sXZf%%zfFU)Cg?Q{;z5~0s%@Yzzjw7jyV&c6&o+Bx6)v5WuNalwR zE=QMdZVjqJ`~)$PGH0{5B(43BE`llku3C9>hHD>ctOYCqW_y545gos zG$z*y^m{LcoAHV4D}LayEe)H+6oS3CI0b!}#us|wVMvZjYE6oA-;6+p2iL$v+Hx%q z>^~It@7U)WgiDyhosJP*I&BkAtPUoP`+X#Oz54-b&QchRWR!BkF{6b)TKmA+duuru zfOQPjHmB`j30$qg!ND4v3ZT7zwqX0Z{gdZovfD@+#E6_5M4Fa8jzwD-7s~cTbxiDX z@LacZc7Z;?km`4f)^VuOh)q#gJ3eiL5BOPCNT&6=V{JV_Ky~zT)QT+}uRyRw`Gn-W z)A~nZ{$L#oxG+AFB>Cep^Aw0Ot9MPyZCU=XiE={J8e`ioK4oYmoO43}Dz#D2W!l4d z1wZwO?voC~@W^*hrLyDSY}7w3QDTj)xWtYzTwNHR;cB8U3U54Fw*ia^A6Ahd#;PMV z1zJA4E4%Z_51<`a-6fo~e+6-yTc~T*CaLH{@;6$X|0_FT%QcuSbQsIk2La}q#XV-g10IjY9TJcB4_+vZ(30BoHP@(Jq7cTA;-++ePBrnE@amS>t z*f^PlNC2x&(gf8L4)oY)bxZNT2$2+?q|Pl)jQxU4!`iM@rbhwP!MLM$&8886AMc?i zh`D+5=Ivkkhn4}JzLz>_^aX0_a?1GhicHwvH5j;#zybd3c?c5f+5rzpqsrW*9Se1eM4pk z?j7LE*K9^vE~ph!*2zc253R?S4QF#gR5D;%Kk(C>`}|sIf#BYUl;WIv8rN$`DNfJV z#lJ1mjsn=kN8{=1P8K_uCP+gj{%cIx zICfk!X$IYv7Xrh%mA>jF0pMaK8CHtdV^1reQhc>WU&ile7>JWgBReIl8X}ZJHV1Ml zR~C&q5AYN0pdrKiDpqcg<7*uwup@}!SiG1H=3L1R%c<#-Hb=*rsy}#fjvd&kB{Y=*MKs5KunAizxDrbwLiKt)*LhibTX@*{m zpB7nI2^wdF0N>vSXnAa4tnAYOj1v2!wDcwj`gVr8xW0-7;Q5K1_;o^j-3|28ScnUz zKx^7guVM*#sKSU*R8$P0QVIcF3YR5F8aB&2L4$8r1UR*PAY=Cdf4VJM?p758iPor% zZNeWe-ru?Q+<*oPF-gAa!Q}#=1=-P6qwoJyCkMMy|93~v`(U_aRSm#5Yfj7qS;1UKlX{Btona~R@n5kkX z4h+>O`StVDO>p3AUVxR|Wq{HvVMOvPUiB4QSP0CU&fop203PyEj2p1AoabUAj5ts1 z8mQqRWQO?JazmzNU{z^?JRsL~%H&(~TXW>#qyo$7e3O9${hoXW%hi-Rh$kqK0(TvBSeUh2W^ z)#^hqu4Hm#3z1{Mqjz2FWHjDf8}B|^WP7H!j1R&^za5`DR^Ewyv|wRLGx$r|NNwi9k81G=>ClA zc+l<0@riXH5#8XqO`@=V|^S z`>Ocm2q3gVWJk4XK=0)4m%283A6M2a2N);xSS|NIF~qNBIeB)kyj@^Jx*vY8?mWoN z^%%xPJ?F7lIsLB24hcvZlw)^T_}QHX&CyaJo$dmOb!N~XzKa9xXXH79J__K;$0uxq z4A+3|dHG#l?n3tmk^=06hfX1FRjKPdV7OunLkjZvXOq!v7JIpHWhrRKM)xfN;Q}%P z!^$4*uv7#>R+oQSA>gm>%w&J1YiPsK23n|5^ME`xgefMls z^mu)`Sl8~Ae5MuFw6ea{1P)77oCKBEcwr=tL_ddeAim%n_kKCRzfqVD&VJxbd2y$4o$sKG!QDRQq96lhahqC;0ZQic~zHR#CB$j;;6iN_uV ztRhv15(d%e%U385GyZ6N+W-g7Ih@f`yHULW*MU9oENCxkLv#utXQj}H3kq+-Qo>0& zQ+FibACAYkPRpri_U{fAY~bP>Atd~#;OGUmUhgu`{muc!Ww>V#~<<`eA>-`N4$=B*nC3q$*KjgUeEOPFazFlnnXn>SXt9&L&An}OQ1!)MjKAL!JR1|Ktl$$}!&(0n0ayQ6i(YU>{S z9n^IJAWROwk>w%j1kdTil(Q^2(8OgDvofzJJEU9(lr41OLr$nlgl#frgIptjh`esWQ9{YKxo?OX zwxEW_2GNIY@1JmNFkSZT;Fqxc5g7BJGV!tcFys#45wb9Gy@HO}*81{oM!{vnC@?4ajS2_((;xz){U6R2O^ydV4gJe5*c4C6QY4atpIBq|h>m%O2I2+vC*!-pW zi%r?Pdv0EARIEim7Sw0G;|3Inqo+L^qZc){lM#}z`))#iFb}saPkS1EaeZfsx6#3C z%b%|MBhMKg7sr3{D`g zeYcnx9sZRj#=u;?_WzRKL_kE1S@Q?mNdk!>%1A$RoP@EV$L#Fv>JPI~W7wY%&Urj8 zF76>|ne-5P;`FR*``Y`KAA>d9M%|=RDDXjv-G#tUEdTArJ@^+=-`NDMyk?HjZ+IU8 zQ3Ri+j?SJfTefiI3c7wX(Q6^)k=sj{!i90jnaV2Jfq%dXq|jEwDG7<3+D&(jZ6-|E zLNa>AC1Dr-p~|K3mgJpI`2YW`@R}UPU+n7Dt2Nfl3F?uzZtWEl1CQj<_nLv29a{)D zi&Y-WPrRk!7xyGns|bk@^VS4ZHJtRA~>l(PR9~EJ{}$&-IumGn)dIvVAv19@{?Z*w~`Va zh3AsiAKitZKbBZ9H}`ql!2aQ(fl>RZPt zkS(V)sAA-B=O&7f=ErBj!+#x59%nrEs-GWWM-B|>yxrGlsLjGeazxhzCWI8YPM&@5 zG11A8Jn;9+Fmr$_Dsd(8TqHz@ifK&o0HfyK3sjs!^ve8ih}mxO%p4(d3GZ?uROoTJ z@5>)PH1HLo4zu>(8UA%#*mQuH{L7ckBvEk6(OM&;XYos^T7s{BxAZ>j*iyavw9W!We4c|w|Sl5Q2*2ucODgU2zjj0L%Y-LjZ?1uq^ zV>Fp0wsCbeL{2l5e-^ZQbsp=G88Q}2#ME@gCM9swz}u<2TiyR&3`wyOFr~K^cPGV8 z!bPM-SS3yz_$C2f9C4o;liaugl07OI{T3cRz9_)7#{JZ-u;>JuEhNKHyAR(v`WMX; zQB(Vgd6H5q8^ZyAln|XF$DNtsK_KLjolW2VKJkNCWdfGxVizYJ9heD;9RI5t()n;PM=C7ty@c!Uo@rf>7$GylC35^sIK6ZrZ{jMF6)Er@flvJe zFgd;c$iKi#p_sRu)cA@H+3<%CcMM}O>ZDsh<@x)U5qyBH^;OZVSa9f!zQEQd;87DJ z3HtkOH96SKsgd zFQoX#`j*$c1F(YYeR4E4_6h)2g?=yB>fMPTfp^Bxd2Ccz0D$>qV`N+~888Zjn1RN# zi};Og05V1uFeW)F?gl5d9st;Yl@Y%m>@a#;>;Vg$JXp(DJP8>aPL{jo=idX(3l6h# zpZ9JP5*k|C?Qi}o`*V7KuEKu-INQC0#^M7Jb9MdBAdWl7Wb|6=g%bS#tMp3ZD#tD( zBV!rA!chC--FeIR4zj(u5B5nBaf>B>oom;{%moB=QCl%k3W-Fy$XA-UU$IvpF467T z%JkQHI>h@Bh^EFziMMcAuJB z`HAE~ARSme!2ekwN$!n*AN~CKQEifHHecV+rvdXa^SKc_kDPighTU!Rk7FcBjs!G! zzE3gd=ZpAJum1u34&pmuPf?5_bgSs_2gUdFjf*wNVQSwQM>t@aS;P{UFSp1f;l&vK z34qCc>;Nc=M|Jdlyl{~%)L4vr2>S%u+yYIT?3!P%eCLp`iGYGm-TE%ap#f6|%YVF{ z+93)MT}h2D=I33w2va*!jbjejsqLUNN;}DF{8=pSs_HZ$9Du250%%01DWhq1_^g=Q z;<3yz#+hQTXS*UEJot2NvQ_1;qU1Us^5f&H9J#an%v9ty|Mh#&IX1%22?;;$ z2Y%)CAt07x1t7kSu%%*8KZd^n1*Wnpk#iUG0gy91CDL?w39{~t*&+AIDI%dkY`bzW z(87x&j=S*Rvs;6BI1zbq15^87{~c6;dt#$qh$pDwg7O}B>2=)izW_HKsuVMq{|gF( zYFOy<-cRqUt{!S;Vd1e8d*WXZfbD3(UiaL(6Uq6PfqV@~J)xufoxMkpg&mSsLp+&;=KUQWfNsICj_$ z%tBqfh0^OU)F&YNOgd&Q_VVRRS&8W{?@p6M66N>gv{rRMaRe8n$Q%v@vbaBSu6O%d z&bmsiqJSzFhq46s6(QNycwCSYPi)aZY_)0(EdN-kA5oT{SZPvDY~+f!L$>yW>4Esn z;Tf=_D@ep=_sskQD(r*0a6o5b+-fF>;6nZG-Z{K^m_q|6)nH|=xdwb7 z)!6S=U;bOCqIP(SDPGkl7f8H0_k<4xKEta(t3q;l?}^l)z--x7wp+6SV5G_r-H2lv zn+T=h%>Qj22m!%8py{SrQ)Nw6z?F%58$>`69; z=MS!kA?jPiUi^!geeZ1Xd{4ulb35rrY+M|h@WK^GKjrlGYsVcPyE0H3H7PAgatLsz z$M?v@{cz7egZRl428;5)uoTXs(j-Eay6*k_pJQ2b)Go->3 z<4i7)S>@lJ9k}}pO{;^tom~!HVDsI7X-Cx85L0M!X1<1LM{FwQ?zUA-+UR*(vsHG3 z`1EvTu+IIN(pQbS^_Z&xykPQ$u#j>f;tpdk+HCPmp%Pa6p@oBU&GYamqbi4y69YeC z1m1h6x;f^;>Vnq7EF4;{IJMgx9it@lhO6HKGk8Wo5|s)-Z|@vEwf6Tbko0xh^8LpT zwU};SbwHGd2X&bs5ip{!uTRG)_S34hxc0t51^|2PJ!|4|3|vq~3zy<;k}rU#L!oYF z|8N&z1h)v!fVKKx_?$KDaS0?m8xa+2sr%2SQi}yo$+7dj}Wi` zs|5<3HgA(55%w-@=*5c~+1022hNu50I!uoL08e+I?{pJo>19od?K!t4M!9=W)KJF< zu77f$dgNo2Jl!0ODMt^Z9d^a9fn5MKFDxC97=O3Vi#rR6Wt-qps zwqbRU_vNa1_p0A~krR}dT!Mc*6D&e-Q)w9(aD(W)orZ*@`2*d}Smyl<TqTOIgUEd=wfq!cF9(ccs-^k;C0Hzk4%i=g6(pyGwgoKCbwkoxx?Y-~xrt*ly zzc++2fCrkoS<17g0F<;ET3+H-RMiOomrKqEv*7v1A3{_v_w~Wx^o_()<>Fh8i4?Q) zmrPy$A;8f=UxP>O-jTnVkO!TKp!-2QahyRa@=yCsLwMgR{)hlgA~@p-&~QUl7Z(>D zIuhC@j_{qd91?#qT!b3o!U!x$BQHo^8~_#4?kMCo<3RReAAE3FH1vDGR?{13;Mk zhy>|dgbxneMnG9?9SYq%VParlXirk2vI4K-!!vho;AADndW(9T^npMsw!?&CrAH~Z zY&`3tZDYLRjIClr!9vL}sNRn4;LuvN5KBacW}wvsMUzVIO*^^?ON1OE=g;sDAdxKuY*%a!pL;DyG-ST`Q-J=093WGUlJpsT3Lo z<>x7nk4186+|JO=Onppc$c)=?1i!=YnTx#@9+!KzecY@%NBo|QmiNlLFvT?pxh%Na zPYm|tpL3iVYrDpu-0BdzxNrwAv{#NL9m|u)LvG&PFU{&gl|vL*GzTgG$4j zkrG<$FBVC{07Razy~P)YlJ&Ar@qFFNf}?iqFu+cE{@27d@sdi0Vp)IUa*iX)0c0EC zB1-ys;Hhz;?c05fuSxmP=|aPdt$L3_@OZf_ouhzf@+1syCbUAZnMl1ti8u80;pkk= zQ)3P_JClr#ot}hwycg@6nw*IlEATO+bRzZWBOGOyzuxXh(|8ctq3byQI{QM#xy>7K zR&<-qo5w1`$vX|EUg~DCGo?R2ycU^e;8inVHeluuu09n@P{PsY;C7%Y)_azLuzVZ|6uH%<%HtHUOTd9c%Ryxohv*r+#L%zeXW0K?hdpjI`(7q)k5Z9u2A1JqC&fOymPy^ zEhYLG=bUKe0KwdFaP< z#74)N4hQdc1$Kug%w9k2fzh<5cuZ|P2W%8+*6AYkP&Sz}j$TG0Y0!au?_1Bc5FjA( z1i|3BJRsdBq_rCI>F2wUmDQ0x+xMZ%@7IY#0by`Yd#ip#{Czrd>l zCp)nDe>++50US2PAL&+7n}JFr5C}%?Ly)GYa*J0Nya{*)jhAA+-CKTr9pHyk=QZWl zP7UtN)Xz10K!yK$nc#g9YJ2?OLzx;2ocVg?&+ey>ZQtq%3JU)I7bb6kW)M7(fqsZ# zEa(wXgUE)X?FV5vG%;&2^${rn6(0io@$BUW(wto#S&j&=RnEb6w#-soCU`%h5gri5 zDBHLbw4HBBRi&3c4bMyt?PY#)q5F1IQ`27uvK%@JuY?nAzic#huyt_UvL>xPKbG$3_bn_L~d1yfK+(-qU@l>YQ@@67SN;)LfME==DXdmQ9Jt@ z5h^xmK1Sm9B9?J}ibdzI{ZctjO64;?E$7Ru^(wQWuD@@frCF1z$;AC#fz z#B|8_S-dT@{SLF*=+7-cnQ8T#Idwf4!D030@k7X5SA>;W7(n-wnr`L}jRk3_q~%Jq zzuzek9&iOPDfp|hKQYA~F0NVl4>(%x{ZPzZ&nt;kAzeoO;{zv!H6$JlDrT*fkfKio^3EK4^vVEt z6MHsdYhBK+l|t$iqn3H2FX@cQ#eAU6b8H{98JD$9(}=qb&9dm9I{&|TnF7h1y1Pz1`{D90pf$CsoSQeYESBwLLE1?aFI z8QG;A5ZJ0`&YVf#H;&G}8~Np4=?GG<*&BS0-GjpUf&yu4l=Lddv2`*OxK94;3A&JY z($9Fea@HMi>^saSQiD-8+j=X#$|k=(khXkC_Zh!g-&bB~d)EM2Ww;-h` z-cbCM*Zd4133=KT$rINaK;LMD(b6118RTt@rKWQ_WRr{XI(Y&9G;gIfL`MD>b^Rsupu=S>Kn2aQ%_v5wDKbOv+>2-99fVs1XPzXoD z6ny>HTL&0trWz;hWy6-5Id6IGWd=CD>Y;SI(cUvpx)PH?4UOE*N z6y*6pR>r=3Qc-N__YeQ^hX@XRhz`^q)f=qYJo-EWh>Gi8Y4mEM0(^KGAziJcAlYS* z85cj4mwM}zildmaIdm!|W%>jSFLt`YP@0qr-8rNZTwJP^Re7e>W{~{J!LO5JC^kEu znL7+sGGx8#D2pi^`Sl;h66Ad4AN*y~4E<3K#Cd+dz7)Rm*X)}MxwpQ&D|;($Tukdd zMO3=G3Hrt)0|zc_rhBCheJF};-j#NJ4~MQy+Atc}rtw!*03`D?Krc_O2cq_rHmYx2 zKnoa#mi{R^89Md9M0UTb$jK7B_@6w_>B6iDr#HPPylO&O_5;r(D=F*Gl!m4SiJAN- z5E%Z41Q4t|(*Bb7efVRaVya?*ruxKkgBw`T#i6iR#+M)uLfC8a&(-dbw)5h)12O>x zmFA&l!qMj~Eu=g75<3e>z2u=R*CGu&%F%Kd3Qz#m{mE)} z=g+4?MW3Q|cMi)WBbRz~I7k!&=M;v_9TU8gdLJfp2Y__96Hey-STamerM)L?6*1w^ z0C+kK0~~G{mATBurRqqFTDD?fB&he>_t6k6u7gH7+Y9aG0?2QICUW&1Oca1wjK^Ud zmRe)A_@(hZ(EH+nxda8^tnX>yipA#p7`Coe%K?`Qkhkgd-5+aA{6UnD)*L6NICusw zzQml?4?WIi=mD5*6Qag;FiSr-cJlQ_FQJ(Pxj-A1GzqJoAsCpt!^`9F-ij?#1P#F1 zPrGQbJ#T7hDOw>|WeB84>x?^1z`%22CvoR7lPjVyD8h$`vitG7ayKU=RV)YPmI&nZ z#d}Mf(mWwc-BkbXq|Egk;d7$ozX9D4uUViH^|EVEz8=gh)~-YMwFWYeM_>d(o}^kB ztCZPb754VTBE_8Vlbp&&9@BP9){QM8j3N=CED}NDpHwmKVcD54$9y(Ve;u%oR?-*i z^bn^w*vn&7>U{IGUCGPloEm8j>DlD5%Y7O09@3dVX1ZJMiyS*_(=%vnKF2&krjY92 ziwKGt5t?kIg8l|6r3E2bu9V-a<}E@%B}8a1gxgfY0OKadUPxT*fsqNhlX`ZUm6yNv z8xd2n@@1bcy1nbbDdl&imyWqedCfhx=`TO|I@j3#_Z+mQ8K&}?@&Ej`DAz2mii*zA z0lU4q%~vC91UP;+MDTAh_vaeR%zo7HIaM%mmC9$f(RjF;N`{ZY8pQVq^djNy4W0Iv z#xQR#f;{LqKU6En^*IJZU-PR1uE zju`>=_Q!ZAg?N5!PifE8=NG#)9`$g*ydi2kMVTZ_S~#)w%+!G~X#Exe6TTVk#}CKM zJY@oX@D7ZqkkHY7QMxq$bSW~&sOhqvKHOJ`}C`Z>|w&!+* zftDcB9i%;4LCBtqVjuQk6iG46q!ldor9wU?=pJ;Y zS!%k^_jc<8P!yu*j6F|KG9RIK4_ta}y=flWdu=NVJ8W%muCMsw?dkZ^u>PUZ(b4hR z;v4-wMjGcO*6+^`^V0F3;e$D|p(C+vS(;{UI~Wn>1|rO@ zlW$YopqKnB&9MCp;L;n7?JQF(4vxymfd2e*O0%F>`Ilxw)6!7*_<*Xq82CW?lT;J%ceP9TcI7~lHLBrywZ@M1dmTTf+&Lw3 zr}LA47o*JpXay=pgMpyBHo)AHg8)t{n;`4cU)ey3WS#6U0UCCM#!$WD(@?7r>|5N` z47Jm@Q5BQks}o(HE-P#{(ldW?ZTpUi%dO*voruYPMEODQEUqo$?4)Je4Q=8E+QdE5 zxiyya8aD;9pdl>>#)RxNn%$FU=si19VDikbYW!}|kMG}yp^FrWcqLT$7JxF7u5ZKv z9NpZC$Z1Ly{EW8&;ToY@AO#G*c0lL&l4L?qjg&=!Kt3Mk?Ej*XY!aH|t`COKmEBg# zqc90F9H3s~+0S1&^W1=-t(jE#bKF^x%A@B>5r|u8lLhF ze^BmshcT8?d%#_|&COHzmUim#n_w&(uqK*bHX0GoYMXzHriYnco(F6_2u;RKd%Qj$ z9#@rtu1FC{N@1s16Q}PpcDB^j<@MEe$7de4K2n=8CMN8fynphAw(twuFCzqVUfr}Z z$an0E^Ib7)2t)i8c0h^|q_Wm7I5m;PK&C@2Q2M>tPu7QpESiNw^allh3*eiYw<45LK*q0m(7rr8sjkNw2jnkeRxcj6;Tm^3=?=bi2w6R#|o+zfqE zNMmXiZZn>&qbJ?)c_1=rTMEO}K=*_MSZlqN&Duzfvgkp_nf) zjW??Ik;Q}Y-L?og&}k~!o@@dhW)9Cd6T2|Q)|%6o7ODAMy9-raZqYb*5UVGz9b_0wrE7DB!T*-PAx z>qu1-(z{50DJw@Z<$XMu9#^eXQL@GVl+vdRdDYQ-;-OgJkNRwgOic5g4GxBT;X=3L zWAig%%U1R=wf~m&1b?4;?z!`hUX4$Dtlk!8XPv2d^OW~L3z+l0V|&x}8C}z@MAUyg zzEZl+m?<2C8AW){-&B4U4W8~g;QpMYg2#6^fM5JMabeO)k@FA)K{k>YXaVNko%>?* zZp8X@=NNIc#6c;y;ZyJ@aJ*gE57y9rYg=6qL|mZz*Wn4T1opNp+!itf!n7YLz*mzPW8f#+nOLer(W?2QilP?xK16xXNe%Zu?L|! zStBW=YJu#k(Q>!cZU`$x!TA!0Twa*_nMieUUlQbq7(fa9PUmq@myV^V-trsI=7pqg zZm5$OXF4hc?_@s;sadk(q{+N@ohDz>9a~ylIG1`2#uOZJh=(W$=je|>aR<0?9Im^m zsiu{rNZ|&|K(6p=hGD7mLu7R%+z%F~xE97H(_bKESPd1L<}eH9b{XpP8V;sacfq^v zJo|ylJTGqjoM<$Upb+A_F+y+RtV=CgSR1PoNliX3EP_#X&Mjx|=ch&(q0aZVTEx;< z* zO4IU<*fXX%-+1EtQz4%r%%+v8cVOsFsBX1TJul2G+eINV`$u;escTURW4#5Art97o z&_bFMH15ORx>*KoH-{2BNZ2*!V3n!^(fdzrlP4P;;|++BxLhbLOMbwe3KW7nFKEY< zVT{!G;riu}59jMvh2$=y-&CFR$uK5W9{i&pjcw;7ZF&S?j$+Bf=4QoWfgAm~KyC`c zd^;Y>3*t-`!FH?vte0Q}8rOcktUO62i0_ObtJb{C$3I}HHiU`pwP9`2P*Ca^uVg7> znC|t|Fuf1)tcXq%0a9Gej9we)RQW2Ppia8eS{*GK@v4(Q6znU|#pxdtb!{$Vt+Fua znMr%#C$znVQ$LG4Ln8W5RWM^ttKAIemA%2Yzmi@=YSvuY`tm@r@a|#Qs@90ftlApP=|luk7wPG$(x8h}5duFgE^&Y=_QX zL5kDfx7TOl?Q7!1oKlfuR$$e!ueaRoQMCA_250LD4{(Eq%+B|=c<%%15$%t*e{-Dz zFmUiGoZ$>_awG;+nZ=jh&##bv>@-i}zj>6i-r$f0^zo|k!f=yQlp<{~;XdCdGBUEG z{X(;pW4({kVEBbdrP^tkr%Q_-voICNyacvmWZoJtiwsKp%Gp%QvA&Eb#ld0FKW~B0 zq9Lu&tjf_tX>K_+LH`81vW!OsbZ6Z9GrqnP6#l?K<_v4chamfNz5D ztrapJH|FiaN6WOa19{Ke5~EiWhv3K45X87R5G}ZS8tLLKhNUihU{u+?F*32m6x~d{ zg7dE!&yCP6&B=Sd)@%eH6FZ~u!plh=}*Ql9Trz_}Fj@`C_VIl%iAl;h|uB^bF6H^k8}U&{KGghSZNN%$Sn%$;~JW ztX%AeVOX5oC_~BD%6Y$$2C$87*{VbT0g{fNpSn-8QrilAfU46PYyML4INdh( zf`E<*8jAa$#uZWYx5VRdLL{1>k+~&6ve$<7kw02{5d6fWCh{y92v3+z!%28 z4-yG*s0e^VX(@){`Q(kX=;Bbw7uGA%RglH74N;<1JOExuFR-mkS<9$d>;v;G^yj~q zX9xafPQ~CnugdE?xftzF$bhjuaqjcybShtIXy5DNa*IPQfc}T+1q&t#ql+(Tzk|qs z)1$rTL#Z{si5M8j=Q7tV{?^yJAt|M@=g#=(fp+{!mg;dyb@*+fZO#}`~d zYpRar?5|Ni*Z$*^)Z;~#XLfEwA);pB9JpO}`O|APyUN+~2#s&|jl z3`JQTut;nj-z|rz7!N$S;-*Eq((@*THJ>>=hTke_u*F- zF03cSAmd#g#cwQ~Uh$Y0Wc+MtC3EQMTWO@~JU~CiuV4D4aV#2SSv=zGVq4dkX!fg#{TZdI8KCXb_ps%KTF%wgnKK z7192^W1_)Vfc8%wR}@hC${FkR>tv()*tiE{IC9#KNLL0RTM$Mo{eD@*Fs1#qcPh5k z4gxK~;3VpCylnKc0Jz2NiR+dY_(u&Eg?Y6$`X9Z^80HR*LdSxH9*&&9*y2Ep#&FBqYXc#x&pU} zjD3eF(#cbn$_i}EO!j zxKe(8ZxU)T_~MAON)Ei;%rxbFz9-ML#!JoQf&J;t{qObUrY7iGG%EDHV1`C@?0+y8 zeJ8~&C2g{8+xB)Yh6ilU`@-hsInm2f)t1)_U?J&4Y{Y# z)Z9;u6tk|}6&KdN`(6x6OhS{=T0CM<$Y8SYBk+tsmU80#QxmT*zq_R0!0LC*Mc~7? z_tiB_&%&6+&K9KWJdE%J)M*C}u3Zrm0(q~;K;<@rcya2x{W|F;oq(FJbnGpvt)4M^ zh5v+6_1hTVfG)-bEzaFFD`LpJ<|`^yr1LCWKX+)zP-LBlV3H(qgUdxu_$D9#s??K> z%HQ`FYCa+d!*KXOCBGdLBWfRuCOesV+!Hhp?s5iekfGpRhL*o5sN+wpcS-D@HDI-Q zWC}DP=r!luSaHb)8lg}xFsQYCdm50Qtw+=V%+l|*HCu-KsRZTCKOpml{1W#i}M>OoP1~Vm5&D7?|CTbSs|H;?>@GkU|oD-uX>b|^JAZkZ? zBF8N_X$Gooqr<~Qhq*3E^gfH00iSB3y(ASMq!dL#hkVYwH@`sXeH4Px9whLAnSJv5 zW7i9h@Z3!r$o~A;j-|8+LtPrci0kEhZO5YEGp2%`Akb8iL1v2nI|mye0?l{1t&8@ONSxVi&{`ECUU$xo zwBoCNnLz_+_xzB?kuNy~(z*ntJ#xIg5fmi7gmy&k=#UVw` zsU)gBvv)x9pMvQP2bOrDu*x?Op^9Yzi+~Pf5p(g}dln6Y$&~v&zDhMVLu_vE@Inx4 zp4~ffH*nh@2&14A$d??Gp}6_{DC4fLAQOwVU%X4rBo8RFXxH@>#?TzQPV>bYmRmU^TY#qpG>v;ug^U0agj}eK8v1hw}`V< zA&Y;{?JnObPZab7gZYJ{NHc^{>3s%Tefqxhm%d5N(M+BmObDu^XXoeV*V(yD-)ZR3 ztVQfR$HBR^e$-8&D7xa1tzhoEn^Md!_@FfGe-HZptoX5f+_7}BP zdU>?bQlI0=E&m?m36kV?;J8#Li2C!rvDX5Wy^2AxLo|5En76M%)V&?%E+2JtbUgmS zxWfHOIZPr-6f^j+9V4B&;t7*!e4_w0cB6<}t$YPCs6WGTHCTHGW!|EB+-;opKa>m8z*4 zadv{J^Sc=+p{e2$OzNbgW7S!V5u~0&CEjRew~K^aNO2f(O%B*5L)o9_^rj(~RRzjg zEXS@xY^EQTe!X(j=svL%q=XRQEJ%do5R0(rO`v2EuVcB~XKEcrv!iYP9>i(yhvt{% zLni?@$kySz-0d-T=v<9KAy=a8=9$u<1l_jQn*m{G56sDwLt8Ubx8j_{nL&T5`)`mf zaGJOqNwO>(`I$HJHN?QiAQ3NAi7VE0O=KGpp**s*tpzF9=R?R%3G2Y^R(@WELt+Xm_-QPg02XnYiUBQy{(q zUMmjwrypZo_K5Fb{k@84UjVU zpL?-@k)If3Q|~gOfCju$HFrfU6YXax_0i0=`u3D#YGuVK1>Dcs%;hU*Qp^XKP8Cwq~|-g5lfcst1`#Co+L#xg>! zzm%`4KgT3!b}c{Ox{DW?m?M>Pft0s^Ku;FFwQBXSTnII|g}l398|*Cjp1dUD+|>x< zW{yJk@;LZ3J;D2Vh1}qQOF@2uj|_|k^c!iAWSr@^g>}C43Ib^j5VYR@JquVvB-s4> z`gRY$Oh-A}%t+%Rh?J*W6F)%Y)(*>5KXrU_%o^Hb&xVa9|D0+S3j^LYIYU>KSv>$L zAje}RdtKwz*@7r_r=FET4gN=Pu%wS(yY!H3lv&EX(_qp9Y8z@!zy8Vgb;pT5)~2?+ z*O0)ZrUQr;0qHXL#+MVnccAPiZ?#O5(63F54Pu~*2g|hlY!M7X*4~8&Rks0RW4t0^ zlHadEmB^lN-)joJi+*|+YNe=tFB@1;*UK{4?kKY)pi;+Tqo;W4iK52P)1a_d9g2On zpaUqH^jp_L>qcaq?ks%%{!xqcey|iDl}20EJ%lL&HDn6z11uSaFs{lGW+(+NTOQQk zP>UuFbqW6s?%Zoo=dDg|$-Jhv!M7IiY@6o{IZ23%TbGX6BtnW!C!|;z!F1K6@HD%g zyaZ3sT+rm)@QL@l$0WcIx-0{}hVNiHo^ZYuEl*dp6?FmSKn~OVkH4WeWUo#wa7gxW zWiT@)FO0`|F`DFQXXxi1WD$1`hr|LE$cgyO{NF<_;0SSF07RDdh{9*#iKe~zFqlCW zQ2P88`7A<7ueqt4`*X|NzcafqP5nl8BoOs!*2IKSpvi{@`H3UaZNez^1199|hG58J z$>TmU12DAY_C(uZ%PJLrb-fY4Z3n3<_$iC-w}Cd9d~v8oFKyZH{SC#OTGYfgZv@l{ z093F8Grs5tNkp8M$B;3E0A<%O<{(HE>$B*FoC;S?Nqs=`{2)12(W|$ed^s=cJ52nk z17k7qOWM25fIs(+Tucmxc*zUo=b8oQ6t2YJI!LgcPtOZKV_wR%nkj%C0-@5iRZ%J+ z2Zz0^6%2v^Jv!H17o5zTC`TCj3S}Av54u#i|Kxj{WaSDcZw{_gQ>r#;G}=L{5X^;n z1?$dD;h|FP<6-Vp5Gtkp=v?^rXKAU{%`WRB5W%<&tgs&9Mn_{GLHUQE`_x#t?Q(hC z8PwOH>z+j)Z2FIo-$d93ZoKPy!3%3h?(dc_Y6ZSI_k~QVe8cUB5AyNRc)fKMPJ!t5 zQ<(Z?2C*bZh)Z32*Vxn)1?Q^@j57tI7`Nx`p|k_I{5k8(9s(9HsZ-Fd??}hpODW0V z!+fxo&R!Y#i!JJdPIXB|nTwON`f0s(9O*I&L0o0w9fZWqz-Hx;NPAC5@f-rC;$|M; zln$n~IeNH8^WpS0lwbQ|z8O*!f^tTarepJIynH>NS0*GLy~zvwbO|M`xWpMSa%St$ ze2Kne>|HW7@m93&;A50a5sN&xB|H{Y8z6It3_4WY$l3E{f2pcliQ|apgWtHq5}YIN z3Y20H{z=$BhOL7B+`}1}Ib`lVc^=bW5@jD)s-j$qgjUce`O}|Pg6FIZT-JKv96Hp$ zr~k2)cK(m5{2bP)nW3e?GKCoFGyrBXiC4kO}RPo4ff;9t&{&nNS+n+HT^H2HM z#lXP5_7W!x6g|#;BP4?$HQr7(Y8eK-eEDJATmFF9Q>mYog^*?9Ei&oz_$`}J$_Kiq zGgrqJbc7Z36Yqh6{rO#)=x0-y4V)?o%8(YMaOBOuY%hl3%so&^I|Hr18TsqXg_$hZ zZ(*1I4{h%Oj&&Qqjh7L!ZnHt$ZhKTnM%Im6R@qXrLXpVIDtnKzLbozPLzIOdVcTwKmPCWKaT(Lc%H+PaNpPWx<2FloacGLEBKy4o`F!Dzw$hW z5@~}{=^X!Fy4z3-t#V-h@asSNlOd=QX7^_!>vjCs3_9bEiMPPLPJGn64$zc^C_Kz{ z+V|{q0;FyK#L2D)*Fl}6n4`5fYmb6Y>Aa{79D(2-Zlf`kHlU##o*wyTiCm<5q1L-6 zHrcE!>pAmFHo~J1$9v;uB+ba`%F!*xPnjwdL+w3w!Q`8xE^kKFVUB)a+}^cGXs^B9 z`-dq@lide%wS1s|75AJmZLtQbtbqLGr9y|;yO|QfGQc5C2CFUA&&SPXLJ z+08GoGqaBP0z1wI$e?=rUc#^=AYd1Y(PxtP{0~7s9tVHUqa5${*Fhb)h|tk=i=(3s z@JU;`qVb9!Bn%J~xOGxU`>1k?bloP1oMGyh!Tm6;^m(4@+2^hFf`ye8af;EL7*@Av zC&qpJjjF=;VWtNlEi&rHy}u$xTm(qA1(ig_NWK8*K=prOJfAResQ(b~;nmf)t{%VC zShIEH!6v&P(ZJ*fst`&pbhmx`3Q{dy8;#9(`Xs+n6t|cKJ&Gz+6T9j`kpGWi?=vF z9NTT?nKl+pCh&82or5=ij{fQY+<}E4;j`vdLDTO)aqWLsjcu_&5vg*A_-%bh|BLtD>pNJ%A!=ClTHUt9haxzXwTg1i)8kP9 z8(V+@G-3fr^Pih zyORyHyimkKZcErsy5j@iva+a}abgP8hSPBoev0>xr6hD2@wo>{I`xTw<{$`yK>s(N zdTYvD@M(${3;&3JmN7W0Bp+_k9`>UPZ^Way5EBDsg?7{HWLv56MXHky=wG3shj!kI3++*Oaz7Vk4m-RRXX_s%UI?mr45;vmIE);xmOZa*1 zXled!r`gzh&J}YeUN?OifA~hi<>%nGQQv0feGBfDerLjm$3OknAww`D4LlZ!! zbGg*d;#)0a6bjb>(Ytk^|Cy7;cTh+o`Xn0Ki(mQX@OH2)ssq;Un7fB$uHggR&jCj+ zy-b)%+N%-+W;WD%z;5ln$6W^EFSvGF>YZwj9rVnIIaTrrdXTd~kh@oQQVQ76JTN1^ zi|{IN@D$v;?Zw7I%UzJ{7Nx&QYT!MxilK^}4BdqzT7T16S_y@0QIicC)>VJ4n-b@( zLN1kE8pi>oz3k*#H;Uvt`6k5XtiTV8ZL;kiyDB#pwBRx~W04ikf9NzXi{+DdRLlwp z3GMXG3A+R4+DJ)$}hYBn`hx*!#yr;Q!+(=ynrFvnIjF|*N?VS zzqp}|LqDil5<4tk0{`Z^L<%%NopD;0DBGLDErEgWl0RjF**rwzU|+!gpUJgp2Un7t z8PhEsTRlE2x$m$?%Y>91iUY75goD%xxw;1a=YijBay{;KG39%{6X35|RE zvOaXkTn0|}F|`uAPGvy;Wew80r5N}5L<~1A*HSaIimmgktt*Q*_|~xhc$Es=<~Km? z?NlsCJAo95px$SteS!GtT`&4UclI188@xBK@J^2caVMy@x92O%+7CZz}K#> z8=5JpZL#fg?DWUKsUj_L>hGfzyMJUVYo)hp{qrnR3MQ-QE!u6STYBSuQkZWb64rQ~ zxX9m?4wPXbNLnQ=$p2RrSnUo^#Z>ApO`|IML?Y!Pdoxh#4(cfWC^>J3V3>y|i*VB45b_)qDaI=97?y;1_GqkiRR8L zESlrg2(w&L7y&dWYvtl)`m8{9+7_N{J6&t2d4M>^fY|Vc4^QfWBxQe@iKTe7xmH8& z3hR**`}c0XpH0gb%^pm>gf?_MgoE9gf)$)yNw!NjZa%nSGJY^km{#uz=m7V^dK~yH zwU!x}0}wzMjo}PhzfcY4|Zpd%nnV;1tC2F(; zKsP=PrXKvhBSGmQn3KP2p#9L&9}W_X^qZT5K$Q%-m>r?|OZz{Rk!VY(_Otdu&30Sm z(i2XzW&U)eU}(HDTVZ=d6;y}>PQR_ryT%$GeC>ZmJ5FF!VIB3digv2ZPQm* z_b_!aJ@NNH=Sm50Q6}7<)x4$6W0c6WDLsUCD zsEs~JH8#Em#Io6~;rog|&NmjSLlt-yTGlM##ySJZi@gF>tMvy7q|AY;{Xtb$^*aTD zrfzBQVH#Qz1QecyB1&o97v5J^P1ea4{tO=+U|cR8gO@8n`y3f|CTst}s*Pg1>aU>j z<)Xg^0?DBbxc4%s?E%*=+7~=oJmN&R)wwf>2?*!!ZP#Ji_Kk1Ps*-M9ENe34Jg;JG z@`fdWL`CLuFT#qTweY-S&=CWJG)MtVTf@mGL`SC{x3gBvjsdW|kv0NmJOh-0NDv3K z2E-@lTF>)W+zkW3u3RHw8`-3vsYVeZ-QcZ1b@Q(Z`j2nUU#;Fj1hOqm9r|_&j8d-8 z93pJd1VV}OlRA4Ck$Lt24G&G4K*cMvLhY~$JTYD*i-*hS%#gt4g9xM?sh|U83ECoe zfk6rmJG(F00qOrN6edWJ65$u~)NbrP@HZCVEXj`PnuUIC;?IycVMk?8lOqkxX1MW8 zC*A7{71a}=_^p|u9c~iRcj+Z|u+o;9`Yyi{cb_l>Y73o1jJ<4p4SPl5+Sk!hie;`t zhYV0_2NoX0v0TN$W{ACKfs?raw23H%u01Cq^W!GF3#2No10BJdGXRx`b&B0hG$VGp zkD>N>HQ`&PkIYued2g@GgJ83ey`+y`VXm4qdHNr-w%?A3e{~>eCI64OxbsMgZIpt) zmeRI=Ee1r$BpSs*`5XP7GjDl&4^$gE5=$Kb4s<&DTIK@|Q0zW=qSL>z2XK6!tsg6h zu~X*y8uW$A=>{q?Ro`DbWtnr;Z`pv$mMcs^){IjWE(oZtzXt{HkS!n=cOVsr1JCrW zU#2A7yd(Uq@A8ElzU+L^tR7vx4-qo!%)aFhk8NYY1EOvBi%lP(H(hmXxxRWBqMn{D zt@e1ZbY|3%3ynEvUWUCYH_A6X1%;YbhzYbKyC>pqmk}u3mPkOI6`-AyMQ>{X^+HzQ zBWRzZdh8m{gG?c86j-5OLHAT*>Z?9#T6N`!6yOSTAhAtd9rcI%S(Ecl)wzBH>0!j? zo?PLPcaKg>B^x-?kWiKq@T4QaIVfg-4XhYp$Y;N#f%pM6RKPO*e<@(cGBoi0>hc%f zix`-RO9~n0V9`L!G?RV0d&_oK?)E*tzGpu=uIL@XRb&T9kbjTHy}8S+emzU#u6n#V zIRSrD@n#Y(0{+owQM$u@{IU%5`0}6gWIzschfh1z&Z9YC5f+voaD|0EF8l*bV22zR zkWH!EO66Ck%r$~94Y|75WuT)0H(&+NGM~n7XLOd^u092^tB9Rq^J!S=OFNU~2(JJ# z^#C$cGJ%m&UFTr*`TzJRXS_`uVxgx`*hUri*T;rRfEC?AQpN~V5@t7?=fw|)s7WFu zeimdv7Td2$cH|}!cKvt-ctY727ljtc2SZU8O?YT-M^=8dj;nhr}tmJkGBvZ%c zQsRaB(X@+*U98w#n@n!-{^v~(D8gIWw659DNVzL;3tFcIfPTo|`2a1L7ZB?NE|ncD zTxEc<>R0-BD=9Eh#o?D2SErx#cg^niYCLIrk4Gb+5$GQ8f?t1GD3>$dV5%nln0(M; zW1FKc1oQg>C2$K5<%)R3Bd+H2s9Zi6{n1#{o!6|O8W=91L zKN;eqD4Jsn(t*82rYJ2Q53f1kXnxZI{B$pC?1OLd=^!#H z9-VYKebA+!hi($;A>JtwEOTZ7c%&r5YikBRC;1yX)^iuAf-rTdK#sOQ_0$3l{sKFK z?q6AtnhW|i-_?SQg+Pi%CRY-Ya{%($b!b`6pPG2!?X`AJ08LkH;Pe4Nex6q-um&ax zrpMt%suRfv%!I)3=F(U1PutpI>r>(=&p!YND5$8B5+5|6N`bM|NzFl4&FIe-oF49UJ-DJGMs;`fM`<-yrD<&~O9? z$)nS`9PVh{&<5IRfjOj3fp)pYs{^D$jErl(D>G)x!%a}Fu<@ke*F4w1IU?G_WIwb6 zJgQO5L7`12BHl_f5Y=Rt~3s zD2+b$--wF;fWkJ%qZdTd?Ok3J`uJ_}SuKDBo?OUaLvIp814~!aVQu!Gy%IqS+ttcCT}IOuC6ZQe>jI76 zIVLM$TYc&PKZ6yWUq^UE@R)Q&;!D_xc#=Ry;vw4@hfhym7hi_BWCK;=pD0~@9nQ24 zUn^IjE|c>vH*6akW(>T=7P|wajKCnF=4X{$1l$(rB6ka#(1(8F!B~?4-Pry6(pw#~ z?6A1t^&L!9AGk2IR@$&2XXn4v=MTb>?3whW$`9)O8AsUQVSNaw!~J@A|HNp_5PF2G z}67W_Tf9FKJ1PbXJQ|6`4N~oOkQTdr`MyPr#eNV##YPe|Ckp3Qw5s>J} zm-X2{Yv6-{B_YMoU&m8YhM2T=mX*K=26(|S0OOx((Jw&$cc9JKy-=zG>nn?XeVb{x z^&MunTBRgM*gpg&cp;qWK`BMew0en-U;Pq~CIrX1e$&v>z*3cMyg$F}NGRWxG(G9s-11Ryd zpTX1RnPm+Y6Cu)&Bi5iX}x_QxVZ&CP#B;4>vq8A^kqL(^|bsp>BLNkg~MSpV+Iba*a3+UmR z;<$pDg$&j0yK=Z-l(Mto!ACyyLWW}$uQ~dYI#-b-%bhd>DBLaV+hS3Ht1pNJ8DC6LZ+iHsT zqHhUFsfL1?S1E`rLE7buXh!7g>!|ctLi-ei%?z&;1D&H~ZXC2ohUB()M-Sj|DYBmV zT(Pq7Hj9*?dU9xWD?O?q#&2r#@c?{h32auT{fC-d{@qIG2d<*f#R>c@k{YxTM3mn1 z|9(B#z-{O;W7zXh1@ZesIt&hQ!!Z`w2}_beR=0o=;67gmDqyJqS5R9Ni?HcBI)f4W zP~5Q1e<`LPB{O)tSrSf6RwTSXyPL;jLjO~w)#nR~1GgQ}b0jyS!5+|} zgUBO*BVV1|$LBGPO#+{dZ1n9LAV=V!a_;o_ z;U^E`ZJ_78F`|D6o$FA~%F`l1L=T%MYNX3OM<#5HyqY#SuG2k-5!PF#ASwkNL?V=- z)FGm&)a#pktt)`0eg*3$`@YT@An|pXX3v(kY2G-nA2>LR&(44P3c596fb8_Tt)x2AzA5qFC+=fR)>CqYwANbT{6DipQr-~kXG)8=xXOPcMk zItT_1E){Cd7O#8-QTAWQZ4Sx@t(*Z0bnDH+q+Rx66hF-2$d%o<-h>J4gAt4f_)G04 zD*D{Bq{{%Ah{lZrX^;HuLos?QfRGq0-u?d3#N0|4InR=tig)&X6z!wDwp$u}=aNs1f@tCF$(~ zmT}-FB8Dcj=oLWj__DAT&?*MQc0`%Mn?Amd%|?=Aapc%|OY%{gP?&h-D=C0+(BL0p zYoBhr!+n6uClvPXbYbw%I}T=tuVOh1>UbV6jxS=L8s_LK^?8r(kvruOT+zYX5sz{kfNuM8k|T3#L% zBabd4V;$ce2$xzgF;5($eBLPn9g{_cPHT)6gur=_QbC4fmf{x0tZ5CHH{KxJn&*)_D9GkhnRUe%1Nu3iV%DN)lGl@sS17&6J; z=t3=AB2s11siQ8k89`XPj;|!jc^5MapE#DWZ}t;<))Q;^VVo%&<}8T@6v;*_!NA(3 zyhHw?$==8AJ~Yr_IVU`NLa?NfH1Q_A+1ei;#EO?zI$OMtzTq*M#XiFeOfvoW%BlC% zp+wTmos3L2mH0Ox#CdQ>q`blBE&!m$;DI!iN9-N~KHZ_-;0o5wQA-~`bG*j{&>J(O zS9@!HIpZU-(*v6m`v-r(C_=O} z{Qf#V5#>v`uWRt{6sFrccoh@Er%)btYPMG&8tIuq{>vG*(Fjm;B>6V=E#3HLy0AI;U@t$in22HfnD zp>^Cj(rj!G9#@QsNlC{3wK0 zBq}zMx!X!xpdc0nU(9@q5bnC-RY*U%cbH$ex7M)}`cDCNQN6(0W(IXA@9mG%;U{Kb zW@W4&`;>D`m~ac0iqb|>R`8@4gYYMa`>rI=gfrlUg0%v@<9%=2X-*U4?CxS@0cLK2q&nQ7KfX% zv*ZDLFl>@NN%I{1EK@){{xk5<%L4Li)G01Uchbh|v<>YZ@`P9VHLeALXTbR>aaC*O-x z&P!L-Ks?&vI*$fpjk3ExYsyer&H4hA7aX4n3WRN~GHcF5uV`z&Gl)^TT|9B$t}|<4 zeYQ#q5}{mAbnMkqD4)H3ADTSTj3hs&PdB3i@G4Vbm~;YMa&&kX@fKuvp*^h?=3nX} z$sGY1T6}u1=2&ZuHzw(rRz#3N#btLT!mf>%>pox^QhcHN0Pok#gXhiz)nvlY-lJ7y z78s%|5i#(^L+l4m7T!kHMf&9l!6qp7f#|1lz_b(#qY;BG%oFOB4DblrI}x~N=2isX zurN9I$Vt_LnssKqdNBOCvqsCX!?$jIXRnKY4e|5(HFe2>=p2U=R2kf`F&_*rQhsUL z{IPCkgfV)3tK0zWQ^8TKB(MD4Q63+2Vx$cCktP91m~){fcXrLEu;#}#l!g)9Lo)Qr zF_xLv5(j!QAuJ=AWHJpoQbCcPNRS^7nB@XC38cR!ib>5-uK$wird$ zHo^|yT;0r|4R9`76KRq2mIrjmU z#0PBM!^5|xFHL>WD-1BnKpYT2gBApzZorw+Y{Zvv_O_HH3KFq{v{5m+NTWeM%^8{oUuEZZi5i1!hXxVq?CNaNl(4WZjeKN_nHFe=zQnBKE-H>@h5gu^m)c@ zEm2JmDTJ=)JPV;z2T|BPylw}7gm2tA7>Z@LOMaGQa$6$LxH2OX=w;{b^m2K-7>2VC=VSm6J+^;EHy=5fJ1pMw~(-D*OcfzltCtYCN9EK}B&=FR^oI`Y5 z7}j7(YB1xg{qb`~3pN6m1JAYjw|8zM$Mvp?T*|}<_yTC$J0Ki1J?!7*hSVczpS0@Mk00Ojzm>8(z z@a3rIa~kuu`rSVTp5%u{?|s3xn2dkhi>wUzF=H9lF`}DP0GTP-gu!U9Qw4MKg~uaH zuKctmaLw@&3F3f>_R`eHx7zMTJM`RPbcr3#_me`X8PTBpZ;3$YW8s%uEx%`5rwnpZI8PGUH)Bqnaf!{CgXhg()Kp`P7$ ztEiOCjrO~6TnWh5V#`30mc!_dVW`&*iZgm*Z^Zj$ti?Fr^rOmZCujH5sjANpvE1n`p-4LZdc^wM=LXTLVog^z<@1!T}Yh zYIEf0I>L|@GkzA|wGK0pAA9|!>r&Ofmy5zabEsjDzm9Q?KoLL=&U#3=@acck2jr1J zJILPrH_u(Sj+BH5k>z+)hU51$-QEBRXNdT~O6?Em1e*Y=G%>iV`r%O$KMds)VsH)( zOzRDz>!2SvJ4SV9+luOV7)jZ_M7LdPm`9eN+$qzqlXa!R9nUS06)4sv>MFp&cZn zx!6}eeuO2f2^$riHcSQ~s0KO@@_1R2<4A`9j{2QO@)`*|_MDFE*PTBH7nmQx`_3J$ zd`9+wa`En6sOcM_&@duU?Qd@r&RVgQTeCavQBN&9O<};NP?u_15DVk?COAnnFFAn7 z9xiPwU|Aeo|0b>+>t)w;kxUqJHwkx^#Ng#8w=gaa;4~NZ#VcbAZqW^sd%Vw+b(JQt zbl9m@9+MEQ@H$=oG}hp`Mk8Ow2CIKRj%Ki=oXpa_vs`FC1FW4Ai9}(dq{!U ziZ3pzB%}zRpg|PPO#V85#QGJ@_=lowF3wt}ECsRz(EJEPoA^94cb^^^3(l-HusrAj zQp$$Se}5cIsRK6LQUuA`!x46S1MdKcpbipC8l%Z~dHnr;>yNl}sz|rrhvt7x1hFHr z${N@Q5DyU1$=E33+~v*|cWjK*7n4@1`sMAJ+g1+tPPSzz7Tl4EFh2k0&ipNN$T5a> zm`AuZPe=)l-3#xIwQ2xno3IT0i3_LNr?zcY)-KQZ6;ygm-vk7{?O~K~0VrsTp>XN; zL;OGxUFG}wOZH|bd^|6V5reQG3-6tYcLbE1C-TiJgyO81YVRoiIGV^K;?vE@qfg*^ zd`o%%2;t3WDC)F6!5n=WXC(uN@BdW3#o@1_I**&_^_e3H=z(gp$sYgxKrytaHF_!H zO7rg%#*_3x6*zx1yg?l+LDpNJ+EHTfK(0^2y8ly>{oK}PHBR3ytOod~;Z+G@PaXVc zfhj2_vNQkG<9kXb&Q9aV$P9$cWyM?<0;Hb*_5v72C@NIO)6I`x1dho@gpd?b2zt)+ z7K2lPMzToCS*DX?k75smfEhx}J|>Yb^gL&{}`|?0L$OMY zdAF0}SS&5OwC>Ru_6O|=P_iQFrEQ~IKf-5~cqU*HaEvHgQ$c+617JkBoOu74eDGqp zKYg=u2X3TPii~0BioQRYp)Xqmeutix7E!h$C;RK^im-nU>LD?TjEp!}%>F18RyR;s z%@Aw|(MMV)V7w*5J|Q0RvsI3I(zVd5DG z#!J_;i(Jk(%1e%-8;tD$gUQQERZFPUgsGU8Z?eUwlhUZSHQu+e8qv55qMggvaVhLs z{Ej#G;q+)hjkA2}%qI^EpBdqvsI(Yt?v4=bexqDN4N$4=f<1p25@Mq{>|AQEzzRvb z6#>3dg+j0X(}vuS^4|%EgaT&uoNkwZE?!ljC}BIAJ;U*INjhkl6Q?xgvTx3s8$hQZ zN-h&5-bFCUr)xL5QGM<18^Xw(RRT29j*el7%knc)VI*5qP?c7asjNsYty}mmCrLQ= z7-rn%+2a&nr@K3gH!JQuqEr?8u6sC2O%=?n93_M)hvb$XLl#BILeE{Z{QkWBqtBF@ znXGQTDRrZxp(#YBR`-dZR$9napWp#}I9ea{_le-aJcd85o9c2Kpu@tG*nYk4z`vC( zKYAoG1?()p9|=^paA&%}$VNlZX&%f0l0l3cmQ>>W8v-IrL+D<5F$s0&4S*i^OiZJR zNh6n?9h4*b&=jcLg2CpkC>!g>4NqhN!QtB;&m9g7TNCWDkV8tl;kJOhL1rj{<6*SV zwy^OTk(|$hk}lsq!pCK>mb|+XD)(p^s8w#zvPqtF|Ljru!XdzyaD$4^4CaE+@$*L5 z>jPY+RLAa=MIVzP=l#aLSyv>VA_IWB=RHV>@$T?cb3{0*MSea4rV-K~Z|B_KR7q*-%%g*JZ%yRTpSG`h?JNuuYJ8?ahY(nW0x0*VC8sr zP#%JwE*ae&0^sj*y~sS!^*EOgmf?}RAH(6=J zr60opg)YALJ>9_JyQKI0@^WsT1L&q~YJ$k~e>giql+$J8*A@XB+Bt2fo(!S{bo*i1 zW=Qa$Wg|~G?Q`$fqew6osAuox=NctTaMn2vnKDx-D+;mQ!sJ%=QC?f^tv;NmgRwy_ z0};2G+6ZC|>QZkM4z@xi!riw=tyLbD$pV&n6<0U(pAC-Y^j@@a_+eBAJa>DU{wJ6a zWMDb<6l&Nx{@UIFfV!wcQNcfOU}W`$Gp9}sI4u<9Wy48N0{>+#0OrlSH!tGVi6Cmu zGl4?%D{PPaA9zVvK7Irg8K(UT<8k-<1FLn*E}e(bZj8cdN86?ZRg)4sE@-Ezzbh{R znZZPr{Y}Ot_$Cep|D}d$2OYqZsZ`gBk@^D~vhl}qPq|OE?m29q2Ht>m9xOip@9st8 z7DlwuZE!MIb5L2x$h)O6kCM@+{}WBR&+Ld)o%kla>C7Pq;*e$ ze$uneh8w?w4{XM}0L1pq&F=;>8o%+>~^* z&pD^I8tEZ63TwGkdO{8KLZmxrL<@J0(%~N3z!6Hluj{CS2fc>LhxVaqGHn>r4-d%i z%v3qZbD8aNfweZ1Tgg1uqy=WWf|!dd^(7&+XVYZ9-TJ*} ztLI@UrTuowZ2NoTbE3est?nfchJ+*&QfB_}>s#H01Du`bF^{<8x`5M+6p>h?AGwEY z8ZmpIR+v7*#CI$X=&#>RL-G$ts4K&=)y>%q z86t2oxk_kjv$%EmQwoAf7$afV$Z00Q&uM6H_(##PkH|&PyI<<&O6acr3ijI#0q;cvi{U4@lUUegLW)6 zg1EyY==Ulj%D`1vvk-;@xA+46sX~-nl?Ypz0KyCTLrlvO)j~bw^X&GWlae+maLZ8GGP_+W zgByujFUXB}zY?DW2%N!;7TdNTh`34aP3CN*#!TN-Ewo}j?&3g4(($9!cJ@fx?gF`q!a*x zI3UWsu@zJjgoz7)(`m>a;Y)qASvUZh`$0M3Hog9s0csCsw~Z9}WZu9z)E;rx1AF z{BY>VfOR%-vp@`PXDmoKiD%v?s~?@05KAjk$^l9ya9*uZ)0lh-WJ2b~11BRk0RnD% zeM{;Q^_Cl1CzmeA(0p<$tE3Y+BsSkTIQ!^kyS3X$t6u#6CGzPKwrMN65i$7x%OXNn z@rs>^G)bhrHHW={Ph>m|R3a0APKJ8u>yL)+WI=4HOgo_g$<*CqwqTEf>^ft`xSJUk zpobTNS@*p3#pLao?LR8mX{}-;-$VG!T^|JMhYciQ&5o{^lnU^bF$V{X=|iqJPDtsB zjY6U^813xoH0Ltpih>scKg8r|cQ*s~(MEu5Y0iET{<-~$%lyn%54KBs!b<*MlUaYq zh(G}Z3O4f-U;bV0VWMQpDN)~~%sX=X{>cwl$)`+|FVPWREAf!{CtM-h)_4f$0}J;f zr4L@2e7|3bIW&}dSFZX_7_=XnCS@H~`Jxer4IFo|v5G&!8bA3K4d~&{^74pBJ zhRi;|fiiKvyTw0yWSO<1LIntmy`uJn7NxG#(qpARal9kEeH0OI7#*9=ThUee3!9b} zrpkJ@iLxBF@6y%NN#Hwf-?;LGX&SK7RR)bw!0Jwr48a(∓!|G^7!zr2F0ut+nFo zQZ4*{7$Fvd_uN2Wr68KYhFe+-SQas9{0bpOrFH{fmpN)w=sm?^9_0pFQ z(?k@1`R=6z^ZX9W5d^3ca4si4@c&*?6sL7j$)|bKA)2g}Dn!3k2AC5+=PM8sj+_FM zle1t=mG3Ld1~KG1B+M64xCF`>3T+KwPqSX#9{miG*F@(u#2o4@s#AAS!-(MFaE7G& zkVV4e{5>JAa6`i)W+)4Hm0D)Lc*YES^;L}ep0x`!Bupcs2MmBS`4tF=`35*wfJ3rP z*E@>`CB!IZyMYa`XU-TjPS}=@*9x-=*e_Vc_)Q_C2{fmt!CN}J{lq6=#l=I_?yc@R zLr%)zwpY?2Co5_E)6U_?8hM<**Q_(2@&B)|tKX`09gqcmJ!^^T)%aPU2qwT)Q~O&+ zC&4r34kUmuS3tsVKm)jY0q2WSHM2S;D0oFLu`q;Se7VomhJ^Vz+9JZ#y<>YEjVt`1 zphx%4qHqfp+6`n_Y(!hjR9dDM?ZUF^`O>WQ0l+OHW@TL!#A$RZRWq94d_{)0sJ8+B zzz_sheE>gS4uU#C6CM$bci>cM?EKvu<_X|VvEz+lwR`XLOA}oh$W`6>O1w$uc`I$O zbECyk^*!_2T4SR+b}RNItDD3QbQ2U_&Mh`X65Z6RLq1Yfx~hPR>amV01>zy_j?bkM!T6x4H>! zUhi)oJpbCD+Y#69LuQ>g4~9KZH(CZrxryNJ(ggUY*}h2tf6#40Pi8L+fqM+1cBp;V0NgLbbaL05e*Jc@GAI=z3#m&L}V99l4Z z;E~*X9TOxd4vTlkVu#8C9)`|JlrU3SwSh-17I8D)^ETZ$m;pSSx+k2CufQtI1~M** zIRwVVk0DnKW1$N~-m^aM-AuOMkCcHla05!!GSN@I{iQ;sjt?!W%$@t+@l(1DeAypr za!kdh050L%t4;Ntp3&0u4!9M!aqm0{sy7;|`==B)SlparVW$u?5hzJDd>j83p6gjC z#lj}QH3>tgD87zippCy+Vh9Zsu)S|Ro$S8wv&@*kuy&G2R*RVTCku=9{$5YU2qDy9 z*dLK03LsRtnJ@9>9}b=~9Xyn_6ya})zaI)JGyaVG2I$cvkoE+AHLac%&|z}2Mpl{T zci>2~^zF{WNe_xR=#qq~0jZu~{#$n##sO(*tY^j68qW9|;6|t<0RWOSrW|(cvo_4x z&q5f^PyHpuG+T$3kr8f*2trurNJR37!XYFo1vl8AJ&an%f* zQtLQy)_uUWGyfvj%?=ps08v&XU(R33+;`b%mqj@R5+9=*0UdTUKnXp5B%NP}0|J~K zKGp0RVIb^!9ead(gaG+>B`vRm=&~pv zZ3WZnYvnKi>Fz2Lr3wj!Vy@8gZ1$g40)2upf}Kuycga;$4;cR1iN^)Pfx!6>5;(vP z&n)$j@s!|SZ8+W%zJcu3wI}M;oCsxV#VKRUs^J_sdOm`=!va*lWz7^!diV?I>Ni9k z)(j*eFA-_!Ms1zn>L5k83*i=21J^F}XIulgC}(kEa+ML9DCC6n)>x1FSUnw7_64vu=-XLl`~h z0ocrMY(Mr@x(tO4Hz#QUq&<-vtga99SI_AePL^jBn#Y(}x4It~sAYWZmaO9_=j6Ih zhSUGD#RD4M&|xRG8_mQQXA9Qz5$2jcS(q#awx>ZxKDF!cOpZ|AfK6O+yW)* z&U@V zkhZ?2dG@(H3B3mJeKV5ZHol#;OfMYR0s>?H6bv8P>d3pdv$5@FB4hv(rU`lmL&9x5 z`A)X`kIm{dtmt+u!vt0!tY=$M?20XR928Nx^1c$HCDv-pHBU~WY;@iw4I)QNoUP|5 z@JBdSs=+F?5ge8+Kay2O*VWScE*O_}b-V8}MK+>9gY4Wj|KXc)YasgyXf*fsARBA^ z2^yXs;FYq_>(=gVxUANyv?SH~JQ@2MuAH-Ys>u1|a?b^xpN-bJ&;lL|!eQ$;ucW8^ z^{@iSQXwwJ7@Ujxaa9$Lm6gkTnLmwKU<9h0aQ6b!{uO4pQ<^JE@4v**b9jQ8J9~#~zD3gC5Z-aChb|UaU5f(SlI>42*%Qp*~8&xWbI6 zdk|(?&2aw4dp!GB!GZzZ(HJ%bSc?{fJQ{Bgk6gGj8JZYZ7Pxk+2(C@g1T&C{=0l&H zV|%WPo>68|r3i{wd;qLVakwH&B?)Rsq%EcKJ|PP}K8fJ5y@xLQ;K|$OnaboUmCyFI z`oow>E{YID4Axe<`Ey$0A1E@QW|#`4ZuI75=!?>@&AJe~X{TSz5~<_^kIvLAM^B#A zI=K-`c?7Df&f+g$-$7N2ljC}GJsGmwy)0uEpy0F=FJE39IqnTqHyh}zIKHOgMs5$4 zZcWU##8qP70->kN+O$s`Nm0CJCz)_n=$6?vkyuCndUlC8K!V-d-|bcDk|1mq2}jv4 z!;ZK=G6yifN%G`7PBDs=K2YMln6M&c_`Ff^Y?3V^v7I_fX6@9VBvla&{YrJ`DkTfeGT6gE#sB&0W$nU<%pr^b8-)8bc$;Y=I65a)JEI3rjHlo>h@knq*q&5jK z^YD3~F0r3~e3Y(}0lUNU%!@P$FrXNJ|L9hx0IT893;vR`{AcDpK{G-dU99H>36{koH>9Mu{={*AOZBe!#6Ico1}pl4Xcee z=>)IuY()m8xm(-(glnCs*Fq?d&A29RmOj1KF29i;pt`v>9NXowwdw2b=npP>YkXvj z%fOSO9#K@d{J3?A6?tc;ro7KoqL2B znYa3AkKSuc+Wt61MFCb)iu`-7zz)>Zn4~*g`_Hk@iigl`mueIvXhKQ2OC+7&p~F`j zd)mt?;d2Oo-JqJd-1!2CjS3k)=bo7*Q!8%YXpG`C^oKmTTp~_MK0~z_eXwPoyz<#4 zaDP}%EP#`{Cu;EgCBRX-$aX#1DINk10*vLN6yVmDHwXl}Ik?wZ$E$ibfz1w18M>^Oa|X49Ab+sS`%d{JGDQf9QCLI`QQ#=|g5evj{lr|?5ufcxfaO(HB~3d~_i$?0 z_6^;`UUQbqDoJTF4Y8%r+^t-u^NFv5J6_(3gV7DMI^H9L@_Y~FAq@~#`v}x+$hXRAB4`GT)NuGd#?XGZh;y{{UP#FVB=5&c&ChOxD-8C=oIQtR2 zQGaA2LH9nwA=@^;p!Vd*&R0mk4sHaO6o=h8 zi{GNs!DL5gRtH7-4XogRk~Id8aU`u$Eag|{wxjOvKdS6{lSq2*(3Urm({J&R7BL#2 zz`QHE;NUn6Rn3HagbbnwHTZ60xYx_MbBsrff(EmXVujapoS^|oeY{CRMy?e)&YZ90 z$5#~-I5_-lJ)CL}R{(*?{dhL5lORTnV)_Y;lrYa*M3gSS?fVpWu+zA=Wxh`V#uu48 z6<#=Jyj(w?q5D4HC;ak&6maf+nZg{)o#t=hW)>sZDHz@vM}?rYR(US6LF4~@`X89X z;5PSiZ>6Esy30W#l?X6H*RpmgiioMQ27=!80q3RpvsIsM)LLs|V}O5F@g4T^2*eE&7gy&%ElB(oYwZI4QV30!6YRIWhZi=(;XcF%GI9Iwyz~ zIDIfsw#2*!BN;)Q7de%zmhf~#$dp7rB zqVpo=D1?ESYx`tRAWeZWC+Ww4!oWt=xO*fT5yOEwS~WZ(QyG$uHzn-51c>L9O}Znk z=(b_?Sx@e`j#aAw2Rk8>*vfs-dymO-=!Rd(e%AkT$-1N`Z1^eIh44z7Bz2Bb2#pME z8bN);>p1ag(Uxt2KkK&A`#lnNahbVQR&;mxD3vfqzPyBq;w$~vfMAUt&-MSi$AfX3 z8y1t>PTOfveisV+T+XSv5+0v3QT(wf@?x(muE9M=xd-hi=-9{nBG%{Cw+#;PAHO+l zOwTHEa$uR_A)=t(7w>!S)=+G=QLPJ1!e0+g7qp_wHtKnsFveI{+9Eo>lU&rmo@#l)(!HVEW=!gLl~?A=hAd^MI0rgTmJa7?(_c9{7=6c9VM@4Z{5a3Rd*28 zl`R>z|LW!tc*+mwVJPI36{mGlz)(HAPvp#TVhue3!~rL8owJGJeZa`C; z;R87w4KSt)8z#YjCadz&igfi=Y5qbWc?-hT5|%FEr<6;*39_~Hq+YpZU^>eS)Y*w~ zi*sL>y>;Z~L)OI2GVVJLg`e}8kGG#C@&Nfqkhy_)LjLC-D*aVu42U4);IPHiqeE0^qK()E#-m)!t!c8|PdKA!Dfb{| z%;f==o|i8l`hl%U03iwb86mwvz^FZ**uGQqq3DI-+xgC{P}se%a*5MEBB|!ZHjxRz zn2K%GoUh-&?cqWQ>4Mgv898`657wTmCuvsUtY%^`gYl>3_c_sxWW91PmnW3_7>+De zIg(wndZr29=85Pk2cT-&2ns7{*3JsSNKHR__KAWeXMA)?h&oT)J|EN{2|l10J*o*j zv76D({-cyiqv#$t$4^h2pbr&!@%@8!EY%$*Ic6Hgi{U~_lr^3&0Q51gQuDsBt4h8r z&sYa;EWiEXiZGi+12V*qAckJB0oP6?PyYOT`VGldxV9WQWb z_oJbYx5W1Ee9*yaK15bRmkU<8hwW%__T3;RDKX=fQ@Xv#pIMa?96q12Y;<8VUX6pH zRSSBkcTV*t3T?3~Dc{(*{nSDnBA(Om>;tKSygDX}E}xoG2ZWFAheIijeMz5nii&NI z2IZ0#@LPs~g_`@K^dPh>DnDLLIBlH*Q1xHf?jH>4pa1^N(jTyck_7|UOT*Im<<(3F zxRP>0!^4CBD@@IV_L9{ShXey=sJPmSMEB_kT1P?at0;M)A~4`BWc6k1v%s4xai}t% zA5wwg!0|~WBAaN?d>h=#6DZ1v7!(ckMXL<0w~mcDd1A{4H_v^uk&{yeP~V-)V%vZd zC%nk4Yp5i@r56+aS3Ogm2F0=`t`|;W2j#$KKzN6x(jjxM3ziIrIm!}+jSuj?95|ec zmfN1;Fl*1y));6_bm@JR6~Exl3?6zFJEmenhLdyFT)%-+jVc`riyCkP!!#BaxL8hj zqQ($&9m`9`CdQ||!*Y?QbVO8}Qg6U_v~6G0EvW)#;Hmv+0S)o%WRinB7|7gW{Ku#( zqwC$nFYMc@Hf6$-C{kbO;l2HuBA-GeL_0o)Cg41`Rf8}SJ$ORFtgLP8NMgJx|N)x@_qM9Hiwljz9Q8ie;60QcnG+lD#v%UCL%<0@nB zA|akStKu9DP;X5@l5tVrro;V`>1OX+7~3l2vO4r1? zd>wjY9Z9M-*Yf1Rhj0hwY9N(jVz~HzxbJi>L~A49F0Wh>qt2xe$~r{XGJ&w7-14^R z(I6GktCpWI54WI~ZVry$_Y#s|4H^K#$2eKBbb3(pPaA=M-itOifIM<->%|wRJp}1< ze^@PlBQ2uPxmgd_Ag0paA3E^cwjSz#j_gzvp?q?w4B;pU?Zc zuJbz2<2=se=uyY;Y-%@_4%WJF18=C*fiz|{_wh{Zo`1RFZN9se`V^b1UN&Qpp?l~_bYiO2dX!@mF;c%|zMXtiRdpEW zpBF90h>VA!Ndx2Kj~9znve?M$t!1?d{NpQl6mXr^Y#3S)U@)AO@#lw~8VWPqmrLms z;xquHqz5S=jv4fdLj4*PMA1_yCR)U;+;+aZhBlGx8$%u>Y>0VF(<_)XWwVi5+i>HzC`3j{*wLp zZ*Bgnp}xo>kHV~YNd5OcNAU6r?7C?`!LBxCRj^0=&EMkfi#|pRv+`Xstj0Y5vRNk080-3^mmc z=zDF56rrGD-3GcN9f0v06Ont1U*JcF)nC+P2D1EHR?VBOcQ&}mgxG(*c(Hb*lC@Fk zJp1pg_P-?v+yw>-&!;s0gl`O9!d*ccV~+FH8kK_Kmk|wEl9`=}2NZ$dAiY zH8emIjVU-im{O1)u*;2RlUt6Mfqz#Mb+Z0&K;i6i&RF~Xu3>Nj&u&hXq^D>bdM2%K zOQ5}>qP<4UuB#1i>=M)Xt3r1KbU>zw_^qdX#;QFg?IwUGhaL6#VDZCSfMwN1c>mp1 z{}=rCS2XjSM<{&)bK$9fuF`)^cnP^L)&@bEMMOeqN02!uUrT**F<=jmrpe9v2hwYT zsn zzf}>)E``p6lRF@>#ug*f7V6=xr-z; z5{>?qCV}PfLKq{CFA)8gf5DQ!KB4A8qR4BfS;qfLC-};LR7yNu|1+=U8ptYF> z@a5UWu51HsyNqkp`So6Zbgm`P#HLNi2O5xXuUdMvLGX&T^G8! zA7vr2N>hPGzzjqbo_;}4d-V@_*HzCCOR`}1P^dng-)0sgA!L>1q4G-J!#eGT?h}Ca z-oZ1-`K-^Im+1kW%mUM%tNcK_Xo@P@fs!b9wjAy}uO~W*2fJ(Rc=bg-OGKN7>VcrW z`1XNV_E%*1#sIa2ddP>N&+udm+fNT%L!6dAmYDNi6i)?c^v+2yv$z5ojMx<~IST`H z(kk?J22V+hg~s4qr(8xRxUlkitC4-Y*L*#%-9*2Ef~O!p2jneIN3nDVCE0Y}E$ zATB)vLG$MIIhIXCCt@(Riegn1jHCxm-dhN*h{Eor+JA6fibi`kb;29P9 zxJUv8^wW6AXM=|Wz*;o|4z<@V*P~0v{GUR_vRk7$GZ4ou=6fV`nM&81T`fOvy!iGR z(1l`vw%e`zAzwyZq0sO-tn_Zm&8tJ>9h4*26is*M0!n4{DD{ zDpI{{Ltz+@xD3ssX^)n6+UD%dH8@}4?H8Gv)q&X@`j*NqJRto$RMl!d0ojfHNE=}C z!G9Oe|2=wz#5yrsBn0Ygw)FfP`F{O1A9B&6tAWH-GR+j<4Rq{u8mf6hRANiT+JVT6 zw&d8$`RmGwh};~a`6SY|wgL`=%x6&*H^EEJQ*XO1oP_}?vtjq`t94~_%tQjC$C2Uy zM6ln%)B*D;6l?@SX2-DmsD5PMT;JM-$if`G2*6qqvvG_lLazkc(L%l$Ck@r{wlM0GHrN@#|WC0|@mVf4kB`_=Al7K7VZ?`1=kVWc@}!Vl;U z>-yia*4J)zjf+l{$!kwI9p}{9OI{Do@)l} zq!sZ#Xsx7$BH>Ibx2$rpq$P^{0LN_upd*t`%40;zCkf`5Z-WAfB*Dg6FWQ4-MLbMI zg|2|QUHo?K@o=J#sXBx-m8Rf3YQ7}U=s`07Pp4%X1K z+BdjezDgq{^PwNiX0G8spbTzwg?+>KfdXWhU}c~8!4g`J1m;)&kr4g5z&E7fa&vt% z=k!P9`NEH+ASO9OeM7Xc<{2P2`V($oQe!N9EW8%#xCOu~3xJMVbMaz89e#BEQ&g-% zcbXXVnO8~}(i&XBnJhI@{2TPMs$j|u_EAQ;isZcHsUo=e+Q9Hx#D66LiY|he0T0qf zxI+{|j@@ph#kf0y=@P7R>0#IFnwMqA543KY*!y{~Vi>BPTs+H!8n% z9%vL_GuvZ5vh*}1wt=xT2E?W#5XUGIca`BKiSPaedM82~z;;ruih$zT^(hs^@tFB4 zo(N~u?JxfDI9YCXw)!={V$`1( z-D{qY=*Gc+JqQE4sFStnPhvLmMZCvI5pp+d<==6IU*~5FIaY2qt@zTv`h*;jAq%J@ z0GC%)7Q%aV?LO@+5Pzf)86GIZKh@5>30@+78(reVco%ZP&8fwYLGgyAtN2Wf3biQ~ z71*Cwiu@RHz)`43MYU424*v#FQu%uU#42;y&H3m)#N<11)RnF}Fdh&OTENxzWLoIT-$|k)lvLkaeE9$0Taiv5k&+}Mp8ezboxz-ks-&lH ziU(7Q1b(-*(3Prpae)2Y0Sg#})rtr%LB3IP1kFgVj6@h9P?cUmBmvEGY37hB>#X$8 ze9$PP@BE+$Z|nZC#3|k|E#T2v9v(*ff1fO=R|6Pw)3)>J9QbjB!R9foSR~#og24|-sz)Udk}+4w z?=xA96y3qN6H5PD4PR0OH3djj@Q7h-BNGE>#0__sbZqmB9V8ZuZu;}@o+5D9c4?=r zU**%7SNtIpAzRqP`N7R3rB_Ix4WE}u<%NHh=V6s~u&swjN>Bu~L5@bggw@5HZ=MUn z@~uk6eF}E@3b}!6Yuw(^JPglH+3>RU0d}TZFL~tl@{siE>u-g$5d;l!8Zj0@e=(t{ z?;%BNfAquZ|4p5qqh3Woo7=JYJHLZA@YjeST5~Mn&P!A-IG0=m*6aShVu%%h81ppy}DQTG}S}H<)QqW?njQ2Y8Y3*1bw}Hh|6xV-^;`P9%Dls zfTdVhN259f4^YU?Zcc#ya$An4Iy}!GpnhJ&no9W+?Z#ZVx-P!9$ttE7R{bl0<(SJb zdu*xK#G*VA8PUtT%sFlxY?=k{-s``CK?r2n9>KRBKP>$4wxto6onb?$$(_OvAMOBM z2IWE2F`%Wt218AM0-C2>@P>x7IUbgE(slDSl6ri13w%1Q+l}-t#K@go&I04{! z2Ig!peRh_^qniYyTtu;^#NeUud2w`)Cygo8w{I|HN0N~LgJNsJ)iP9DFY$i_fF!DC z?>(reAR$=Xrg<&(uWHh-$V-Rk0{^mpBF?716KQ|q%smE!vEKJ848fcR*6<2E*rBJq z#b+Gu$xuX#plC6kAR@!ZXc?su0OamOacwg#! zz7$TrlT2g-eGh6p+4f#|dTa;|1w{TR%AICOEl3x6!zqet^QmV|4ZrH*d!FerMI7#! z?!D_SICh5f^BzIksqCnO5l2#K*$qC^H=ddILzLkLnpQ^H+yxf5>kjy`{sCU|m z)?aSD=&MJVmeIG`kk(GjAu1-p5b3suG&MH}iyc;9DEX*CX!8aCr%(007gXd40%11{ zZy|!uB3iN{cvU}VJDxO`6ndK~f(}|ZFpWt4C8fj@MAsC#rlfAnK@J}Zv<$MwV9K#I zYC%ouq>ap^w&!CQEj_|CDIG8IX~zmHS6!f@^2P3nSiUvft55`G+k55I7sXE;kAdE$ z+J8-w-T~1Hp)k=!!Q;Sh!48nGOaob8j1v+Ae?$;^I#{?%{%`Q-22=QLep?co%BL+Zk#z^`P83j6g3MeBUzK3?GV)ImG>v=Ocl z@N!AqewlO=G9EH;*BUpjkB!=dvZWay)OtNnRE9~IRSCPUH6ZEuB_ABd88tCSPv{t4 z@rxdBh5DP%h*SYXnJL{q)wgu8Nn#R)7)OM)R+`>M5#F|C{T!>$RiEm84#qHh1S>0a0r<1gSPZcGrQ?0*8o6|Fa!LOal% zXn_cVI>@67qHsAUFJLMibJafP#`Qi3vrf>;5)Qf>#UWnqu(tYUjinAa>d;@Bfh%B(lz?PQ zf(?TU5-vIDE15kIzF$wF9Xb^PD3}V)31a200=j0bh&L?+gJzh`>%YPVBZv{eZQ1J! zlhI@qYMg%L<_!1K4}gpYO!mj@W1?X0dGa4$~w1t`GssWTZ zh7u`XOlINAR8wt6nsfL~{sAIa5`zH(2m?GAGX+~>1d~hrj2EL3NzFU}Q;4BkTRKgk zgkydvHIXprPn~U9G+vQ>P-}8IWs?&4>ykyhAH1tKHpskBwVvxyPk*k`k@Aor5H>fl@qCn|+&=c|*NP6n`bF|-| zG}gJ0pb%BX)t*|fd*lO#hThImmFuQ0ib%D@iMT!ZCE4|&tZoQz9~U1h|4Ju_TJh9O z_Jq8(I-YtZw0FwT8Qh#RkNvBJ0s#XGUG)Fs#?pr@gTf_F!~c(MWDt!6%mg=Z`@M5=AXxx2PDLHO^ar0-4K#fux>-RYC{6y&uSvvC0%40MJ9o zO3?wl_2GxrKdD2+sCyZt3~mq>ZyAl1#M94;J>3VBiwkv*0CG)eX%31RsA%q~iPmO_3@>U8!zT>A)p-M;e$_{ndL$IJzs&-GF+U}D8 z9uxJ~D+0F7k~hD$4mr62VK03CV)C_apHucyndH6_w6Z$ZtfZX#L-2&XWF3F?x zepu@!IA`r2<;_qu*Bahtc>8Y&XGjWSJ$(aU)*?sdlLinrwswGX<`O7OzzMTMjL=xR zY5Nh7P{;{MBolHGDhXV9+b=O*nDV@QOz7|9fH^6vO? zX=w_exJUD%_}pWJQqr8teU%iuh9Ch=G1svTlW?DJ`HK5sjh#zTuJ!gg$7BZousEe+ zPp|^dc6~Rwx@Qp>^Q(i$lauZXZM0YzDQf56QXgM^JdNB*aQZ#D9zH-v5jvS@P7tnG zu(phTiSzRGgGL(gDTYo?ebmSwh@2=szkM%6E1RHRqYxcOBYZbJxLR=Rn$t&|t8-JO z@tJII99L-X5alp@@L^O1>0>2j^XyjC(6ze-5*3c@k}-dE?WtlD|J<{Wi+`P!^aJ!Xy^og~5cFR!{Pi-54{$Fw8}zL5)~E0`gT* z_OT;%yV|4{&4*ovtMWo{rB7u2)Ch$4^gSE13B^Oti)N+`uyMX8`XbKQ$tWI$-zK3< zat*^d;i$jhaW!&#WmNs?WEJJnD~qD%E_+}z^*Hrie)G3FzaK!DyQn2fBQ~ zkgGMNUq9$0!J~D+%rz*B<2|P9F@sMD%hVf99KkJptnl1OH876f&SW#gfUVYq`=P~$ zI%d(gxyL~6ZUF}V+^18>b2koS)<>qQ(XFpMe`E)RZQt}uchpxmhz4ITU`ByB(-a?E zm=wH5%N+l&rjVErf`lkeIX(EJuVsY?tArE7YR^53py1w2phH#w;9a9tI#|CngQ%xT zGqy4M&kI*)=2c)~WCn^}kmIvpxxt`{y^v2jsY6gp*o62Zw25d^TEP)(c)S|$V}0f^ z>vQ=q#y4Nr&F2iCViPmZD2tHP$ARNgFYN<=^p_Yd92Gb)L5Ubo93ik`(}m@UO05Lf zpR&P6VmI@YNO~B@63azxX62n$J`I|4o6*nu+=(f2_K?WS0xRl|C;3X|NamgGbOPx4 z4B$80BW4)U(EVS3&}fDZt9u>L@mGCMzgx-_b`caR?D-o!y3f!p^Kf;qU}QO!n2c8 zW7e1aFL};IDBR9xoJFWVv=2~8uq9JKd`Yu8{iA-3mlp==0V)~J8^UEe&Knu<&kEpn z7W`TnfnZ?^wE%DG<~PiU0~8grgRpjx_R;VHbl0;O@gvYEoTp%VBtJ0lAvc{=`)x=V zVMJD2`jxm8xko*b)N^qzLczCvz=$(%o4@AF9zM zxk`C>Sy_aG3!XgwFcuC_>TV(FL(i+Zyg$&{U$O59TB%KzkFx)&jsN+0Lk8BY{VqTH zGBr}7=mTB7^%_u{xZ=zZ%>am6jr>-x}OKGFf&*{F^b-Z@#hwJ(JVAY6qD{&`?IJ1o@kv^JTW1Xz@q6hLyq#O>`oK$3uC^Ow>6_V%etYoW3M;u;<}ISBTnRElxQ6 zaIoB$5nR!_TPVR2^8+kSJ>*T*^RBtubMM{(nAytM%z6;bZJ^6?+_IvJ#Z>3((#dIr z)>B!h+$4@5WAw+9Rh8g%HCQ5dd_fIQqVVoPe1;3HF9(;g``M?v&iP%JBVm-io`kUK z){TG;*qV!|CUriAI}j7(^S7wIXxATyKX3Ylbl@G&`*JQ|^*hwFU0(T! zb}5USUymai`*Y{lnqXA^Neok%7nlUrBNk_Ld;b6V1^)U>>=`&}wYZu7;h}4jK>jtL zmkEPf$rT>kPY!x#z8mhdmxYh)hm9hoi+5pEY#4ZKaou`DXU^47uPx{wKVJ$%GQR`a?GO-6LUXPZ=Z|^w zADr;$e>sEeSB3k+ZM52?jt={9xiGkLdq69Jo2a-9{x57Nj*V*L^(eC|<4jGQ>tmA= z_Tvk;xktY-WyBd^eTz_0#;}8&u_~hXhNHa4B9hi2deGffrz&~~Z}bvFJMm7{t+N+8o~F8Z zjANfC9)E&3$Y=5=>#|?GO|m1rlwFt&GccH=z8|~I{bFk9<&`s8MlX??7T-hj_4YSj z6`GAQ^oof;xL39V`WCu#BHZ(jY~7Xwg0oDGYxbg*#@$NH`Wl^gS66bTIl;_&%I9EA z)Wmi&4?KRadjHQ#ok6jN#h4b;_|1y+j3??@<91ose)Y=z%Z=~aP^Q6G7-3b7ZzZZe}6KQy2U5n0o|4G?~R+9kXjm%$c` zCY*O!54K9cyXtK8AKUcnkXpqWx4bG31D|?aD_nA(_PXM)_qB$hv|oAOX8$e88|RrZ zj@9nwZuRMSV-V$14%U%82Sdn<7eZKg?JczS5f#1i*TDD7wpVxSSdp_aOr@vY3Y436 z+nbYzse}2ZF`fYGa6bcS8IAxr7*m8h9@@}_6>aNX+9-i$!{x1b6zH@nsZ#o2*H@?9 z)>_@~B-_Pfg;tYq4ji00&^&61(9$LxzrX&XM$D(O&f0eq<{(_4T9&lTIppY|3p48p z^o_CL$kcZH<_on-VUlr1m1@p045JRi7z;|5bTvDpBeDJYv@by?gI4e$hCs#Qa_Iws zK^=Vt@cR=R#^f+m7{18F;PdCAgwVq2;rB|@`pq@A^)}c~lU=0GxMvUR^-SoU^PWCz zWVxY$33FF)0;KOZd9VSeM+2xKggl9&9#s|~!-l!NL5yni`KmIi4G#Ydo(sZIqD-`^ z#oPhZ-)XTxvs*HL1NRX@d&{litr;Wj<&3XotG$UY)O*|Aok?zqWeF?2N<1wQY2PkG;Em>GMZ zyU_8=OsLhbmT!Fe+)Mj^>^K%!Aid!UB0Zrr-ELR4)P*umr%3^amfa zbT}26pU0Y&?TkDjG-iFh{ps5a@|clhl+}DY13kHKBZ!MnoaRR^LN%q90nUM3U7lpk zY%ZHny&h~PqRt#Z^7J_EAU@T=BW8A7=ePT90l*#5vk0QyyM~6=-|QJx-$C5)r?6Kj zmx3?OI3^9`=sW=!Uv?!lajb`ML_^cut+Dew33{l_H0hp;W|tBYU@orSz_XQfdmZUT`P`vMuoLj!7`)s+;BoG9rh({E6;KJ zQHum_s7RlJZ1QHmQH9ma8yWg`v9Dls+^6-kj>wDuVVDNPAhN@*Q|O%H3U4X(uK9v2 zn6+`ltEa&YeW4OxkeO-U;1ss^C^V3{6~L_$wd!<}$roc5vABf`Wetrg$35p{#nJ=Y z9CtKn?FtB>U%VhYemuTl<9-Yq>CbWm!sH%?1(e=l>!v-c8p zl*)l&Guhw1qyL2w8BL!DW z>yR5ir*&}?vT_S3UK}K@02jIt$Q9T6o7BOz)L#m}U0F;Fx1~%@Ao=XBy?Ej=3Qcz5*jECE&EA0bQ7EASESFoOcI9;HU|LHMSojhCx&)L5bW3 zz)4XMp!FJMST=dx0C<=1hbMs$t3(uax?vYklPzvCfeCTq$neS%{B`l8yi%ZHGVxoHwD ztDP|pz00#JU>4_?GhA%AFYD{h3hD(RjN7%>l3}__E{5BW1V3uKRf?I*n;KUk~Vy$?oj( z15J=cn?ZXNQ(O9*c}KRiIQWlGqam?BFYupr`Fk;*SD&e&@VN9C|CjsnUmvE}VO2ad z*D*Z}2&tn8pGj?rMT0v7cz((_2VyJBZ{^rF`i%ZV$!iE1<}> zVjZ8$FphfwdS$icSUuXHAZb5hmSMDI1&zsm@3hr>t2J@tPfKgqs*j%SG6~{(F*A}H zDU&KEOF8reif$BcmyV7nV$POw?`bo_%lCb&=ur)kwD7uiDTh^^j{R^$C*SwLFFPC; zvFx-xj}!nyk^gCtN&p_d)KPWZ|D2sBYaW z1jg3ic51QN>CAz%?_%m1YyhdyvGT27ZUU$JunOeBT zNwnZO%HgZ8KzePUEm$@g?{&?2a`UOt1hXHI z5tu`gP!-GaJmt#^y(mHVbK_c%c@p7`T(4@;9LSc7!Dk=;K6pXmK=+a!SE0WI6~K7@ zmd_%9Hunf5S^e_T<(DvG@-ufG9D?dJo8nqJjt@(&c7jhX9#yJ6m zzcP7x;-IwKnf>;mi3!%S&v{flMlDz-Bf+9L(%)hstf5#vMEk4f(*%FJ>spnNWCMpe zLzM^R+NJiMI&){1qKbL(%14&N_PMbOCT_HcKqwT7UAowmk>H$_(P=4OXb7orQy&*3 z+eK;e-0erZj_NDKRS{t9)KWa}dB>@&?JPQlJ|cKm!Q4Oh zhF0bQv4w@T&Xjuvo){==e=7ee8WU1xSZ`fc1@h)o0&F< zgz?)=rDUlp%_$1T&s}}- z68z58yz(?rgtuPVJ-ZH3cCfZ*J9q7ROb>k~g>ARE<|OCc1fk&sr`hNvoku#kN~Oln zSc`uSXju*e7nQ_OgCWk!YXusHzfLGwdyjm0xdH&3b1t`|;h>L3fA|RnoWrW7wpv1P zOqvQi7KX~(4xZGN?M*kB6vyogFYJVhlbkdP>X;9OoiHQocBqocd$KDsNvso&#mJON zq>3me0yp`><4za1u)!1Hx5?9FCXrd#K0~f@CcJRj{W;N7GvGwxj;SW+K;K3vFCG>; z(;OB$T=*nT_4ltAF0CNr4(mvZlJS8pT%s~&@nmA* z+i4}K8O%VM&pOCT>MA7BW`K+N7?(uV1}(rV^=uD#i} zo%uR-wm7oAsFjI)D~S&L*@w^D^kVYVuZ>m;0YBcS%_Yu@Fvu_tTsP3WIjEag<7;L6chqb1&C z+P^USP9PZUgh#go#rhFr_Z_HE&)a`+dp!Bjf0dVL3zIqKV-5g$;vJaVkMwhf9iUef?(T zc?6)G+l@hNFiO@yjB#X3_$^X)qe9+|`8JvAzoq=^vs}1ObJ6Ag3C+lL8^Q?tyT|7; zs=sv7j2FK8^yN4akN? zdup7zh*KzZA!`bcq0dQn@No&%M#z?=A&}o0W25wY(5C&-TrUtY`<%aUBzvE#A+3E9 zm%dV$cyn)l7!xWUhfTlA#(9uU7RMDDT0vyUkpU`=JJ2yN$L)*0CUchA&9bFf6%H(} z<>K|-4fyzN9%ihJ28ai2ulWS$FU_YhZiY(K98ap~P6sP8ZzD?CW-(hrhQJE)||e8jn_sFawc z9>wQwd}sH-kJ3JBO*qvhe#oSwIg{*;xA8H;Fwk)rw@Y4oW2?Au=@xi?ZpTO4+U9bD zuVr7Qw;uoWOWNfrZ`_?HKR@MODQ>*2>IB?*Eg(77M&$y&jK=XLE9QlL~K@bqYKTKkm#OOBt(WWqUBf9 zH6Gie5Ji0aCbpEup_4OKaS=-DCOJZB4ikWDivb>R7Cf<$we(qos!hVJoY-N^+ArQ^ zwG1;hjj$atMgmGpSDznMRxF4jXRw9(3h$2fZ9UJgXr_%H{`m6-q_$h};NEq}5sxmW z2NX^Op92oiT?~Qn!$AImpB2SVOTV3;>)4HcP+rn!=ibVcQTibpCVM*TN}Xxlqa-*O z0tuF89>4^=XuB5bn+uBOw35i|Lol{F!YiK6dkU4@xL-qjtQg^ z9swRiY-#A}SEL$GC%c}?_P&;~yhO$7s(BB0=46@IouN%7v0ZJAHv(SOOn) zD+Siqw*?E2`j`%M_~xl}cXeWzRI9J3AO#{dET{O*eq?2TKZ8>cX1{OiWBz_z2_Pk= zi&63x%f|@Q=8&<9vAAXV!afqn&DVU+jIBhwgVE#1Po_P!RVN2e6fgNvIcGvgyGP|x z5eI~%QKz|h!Wd!!zJvKx!o|LiQ80A`T4gsLBGAi?tTRSLEEx_Y& zP&Yc#w0s|Ys*A72f%vU;dPjnVKK6heip5Sz0LoaFtSjP53=i!5Uv zGN>tcAAtJXxr*IdH!T`}ZMVZ~I4ffX<+WTEER0|nbAleN; zTkMfe_X}8I(i#djl;6^(U#GYh3C(l$4}M#J)dG0acc|PtD~HhT+>GJ`3r0c6=LyXj zFv6~c*|+4|B)-aj@DF$-OLWf_nR zWL8rRHy)`iwqgTdq4!v`^g9u-4UzCI*%XZ=Qc${0)0e{Q#UhsKr!_4nP!m^uykePN z5(XFH^M292_owm!UTn_u>{Sqqn=FkJy>SGZdAEo>MBYbiCuLLh4|wmczLn*egV<37 zF?DeK^`>Txs4yOx+AHR$RL%Ay;}_0z@eby}!aL22R@~&C7q;o$9m-&mD4w^tV(G0L zeN?EtU$HAZjFw2-okKG4VV4N$8~?Ccxi#fM(yxkg`+^dt$cT!AkK-z5wwXbuwUi`r8DuN40&4TW@0XL#jZQNGN=7gHE)@Tr(Cj@mgVcRV9E8pelYvN!;ynajH!qo}5 ztSgzRLH@a&j{LCKpndQdu7VQC-CJ!~V`S%EF(j<_r&T1Cd@hbDB%}v5)MDrQks|i7W_yW*A;H2AD$=9G`09L6mPNvibnj8 zM+V!j5rtth$B4IBHmX%`Zwne79c!CcLNl4KS`M+qyx*V>NBxWXbNwK8?C}5H4s;lJ zzX0>!Bm6Yd8}ZD0DTa?3_J~8q9nMPv0$zF+=v3J6K)E{#??0+LLcw=}D|?s;n)YFN z(OwsvKiCLJS3l9h!Mc<7V__{T_VA}@s;ui3ntf_o#Ewz9}!@vh>v$PI& zyf%dcvIq%P7 z{!82Z5fZRR4kC5zHTyHDWIX^Th%|fAVcQCb?&?txK~V>6K~n^?Qz~3=2w=%6t=)B3 z^Tb1w9FrdGgGy~FC+i@4aw(^B=YbrF>7e=Rn6J#4+Rh7UodJ|yBXt9jt)CJfN5oRv z57pSNg=Ni4lNR7IKa@}rAH5fR1|=L$dqzQHY~ddbEmXW_ZMfEE?ag2Pci)Sp!B%)5 z_GVT{PC4n3H$OrhgQj2&QmvrwBFqfVoAWE(3lT_C?-X?e`#>>@Nyd%oQF2Mz(#n^G z2tZVaK!bvSr!SD#IWHf~1~4tei4o~m7)5j6fmsiR>}QQ+CPURH+qpgmj}U}kK>554 z_*0lbhoRg7{Zy-bHs(nj?+XT(?(WKOFwVB{c|{BN8G1Mbg+)ou15@L~y#mM#hJdt~ z4GG%DUC%Z|4PjISycPrnk9o-IzC)w4hIM~uX^{QncQ*i=vmp>e zPEqYttcB#$iD)t0*W)4Uqr5)Q5sn}*Ew?6fC5jJD60)U)S!4>__CNg)h)PYA#>%OJ z=$F2>vCo`_*S-(vgE`og>|d3AB8trLK-!e{>9u=P`S&0}loI8)1TvZoRdAtX*=dF z%~`_C7{6#lQr^@*xu29`6)Yh_1Ob2=d6lXXMgN=Pt)D`KD<$|nk^lEwmH0g0UYvbt z4!W0}u`$O~s*pld5xyt|EX*R{dNIbe>@TmW*m}t$kNpp#K`6G-QVU$UUIX^kx)sc{ zA zYYHbN@6JR^XhE>qwov~8kW_q0UhoxB5t=Z(YUT18ipuySB&M%5YU;w_U!D$NF9t-B zu96MP6BmX@&=pqP+adhG+Nd8yrJ;(!2tNHy(yzvM40WsPP&b_FsgL+3r#aS4tl77}=j!l&zX zxSgdSA>39$JNVW(UD-B1 z7gpwxR1EHl-ZJ`a9$TGYn1Zki8nhEo5TkQ9zM?bf-n2Z?dAtw9vIUU54(R-F9nd-e zGE2NtqM3Ao=Ij0pFIucBQ+pQJ%Z)oQkHI&|3&VMHu3slR@^D#9F1N#b4UBi&8TQP0 zk4{%I@|Z=XTTzerXP;I(d-;6I7mI1~6r9;i4`J31B?X7!fc^{AaOmij?--?$S#4En z;)a46#`wO-l`Z{s>n6%MSH1q&$-e8THoe4#$J!9DTYMj{9?0*6l<#D%nL$HQ@v?Ep zcuzE8m<9-sO|fkW>xwq@UXOA64vY)yRg>DMAJ*Dp%?n;FoT!QxZp(cMb}eVq7c{i59d2q?~_FHrGj&b!~| z#7853o!EFIa~aNT)$W4$k_vKL0i>Bx(3Dsx1Fe_eA`%8W{YuN z{wN^a(ln3yW zuKNv_+k3>pOR7g9ufyicS#0N_ua>hBdtbL-93VK?)2e-AMU`|2QkA;juA>$AGGS3N z?-8fZT#ZUSrpKQ1;b*J1<1&s(W16=(b{{QW!uYNE(q&$Kkg(ZEZb>)3_byc7S#7qg zdaJS_n{j*31@UrR96}K_>LLh>o%>74r+Jf!Fq^g3o<~xD537cPvcikDJOhhaUB@om z0#GvX4r%p7=Aq2T5aI|uBg;w9^h=|tLJpGKeqw|z?~=07K!rib^9Zz zo)3^M5J>vBCHvJA;-H53=ef&TIl zc!E1{%gPA6MPP#ki=G%WhSn4-1sH&k$GvK-wRQJCCpwe-TQi!d@8NCI95OCSd@^U* zJ2*nklrtr9sTalhgx9n3UI1CDy=Mo80E?ilA8q`;C4CuyPaY6?C5I`vg@>XJOtWw~ zy!*PW1vWNTSDrN{9&RldI!riaEYN5RGp?{jpSS7vGoJ6+2J3PIhe%n`rIpQi5Ckq) z$iAUoL$NIistop`A6 zaF(Fj$JCuac2Baw!XmeTmKi3EIc3nQAlKn*J*26`DVU%aYfM=VSMs^KE~KY{5V;=q zX<1f&1sownDHjW^flEc1KCM63Eq)Vzkuj1mPc5)SL51gcSj^JB1=-m_vDuFNYht8u z#9)ELmQA$udw~dRtXh7Cb5<2ZkoJGeAVLtp>oM(sSdtQo5h19GOgwf4#sEKVod+~? zG^3wF0){Y|`p(&wF*|d7gMP5RabLG+-aD-=`!E<^h@H6qc_BSEy`s#-4FMDx_Bfy? zr1%5LdtjIYTK(ngBTsUNfdzZr{YuLL^zajO04VD`7vjtkgu)+FCOueL@#u20b1YIbFJ*mR7%z4iN zxZW2xq37dJWjO|DI*D8nHTW>trldknG$Ekryk-LdO*Cab^8J5JJUU~G0K4xss0<78kTT-}kix-=)l;@WK)F*m! z=RnPpZ;;%Z`^n3RgZSK(5+zW18E`VagIm>retkMv(vorDTq2Y?(O+}*M638yjaF1R z$oC&NjB}k@6l3IcgEf%R@YYkM6u&7aUOyiw{SZJ5D33R zqz$-ke7p^Dpajx7)osMti9sq&Qy_))XBBib;xv`Erp6#G6#kx@CkM2lyEM4M?PWm1 zQd@Cf9V;uY!x2=y-XtO)$9H3?+sn91Sxkl|5xUxaTX=VsdhAj*l$w)~xr`5H6|_No zwVVphhGn&M4+D&Ad+io*M$9xhaeg?{dx3&rFZAnYg*0Lo*WOK#R?j84jL0&?|7vPd z#2WMV+I>%Wb_5I`ZXLtn2!Iv^IGurV(o$x>b>GI|I zdFa$b;Q6BS+>4wfxkL{3Zs(xgobkhXrC_Kl*P~RcG z3#%QQCz?$ktb(W>B*Og4W`v<|{rG1m8=%ystrYbsq+4c%{UlOyl>DAc9g?G1*&vXK z7|0@5WOp_?(e|y4O#g|a6KH~tB80vp%6q0VgEJ&r=HIjn2%6&;=(FyOznYd!B`vwWC9)pl)+ zjLojn_TtCduG*Q)ekq;?X{y3=-poqUrERZsz8qdT4#(~(U#zp~V$GyIS0Kpo6=$uf zqXiNN;xNYu-xD9yaGFo{Gl%)%eCaOramhGnNhFl#VgImNY3fv?*?@h`Rvfi%2TiC? zP%+s z4@;WBS@EzS#G)7HrfHzSzxXK%7?Q}by?SKaAaUM?h*`Om&i^NH2l+UM(YA@N7+^lg z9a;o$$_1ao;~+V9!EU<@j=WZ&DBUgmxEG!`*oF{*v0fY!qqz{RD=TZ5A`IEW%}kw# zguV(lnSMPFEm0GqoZjpLcOwYFTxiw?G&Ysp(~y&GX?=a)$C?mB9P)8a-oEBvr$DR+z-Oj%8caK}gW!Cvr)AfEltWm_2QuSO z^!pn>TJf{-j5y|i-$ELI#fmjA>bX_5(!$*KW5glRu)DN=;8z~P`5_{-CehNvA7HDH+ea!}c+yNNWRPyu*i%-*_0dN7( z$9R3oZnp8WuafL)2j-xa0o)505SloLn+76M1K@7*Wyh8r)JGq|7_rKf@qavthM_#j z0Q<7}#ia)wVT|RjX9O^$$F|n(T#?y&1bEu`TGbZw`cC1!ePXnyL{EEJ+1ac1Qjn&X zCASGZ16mtwt!FD0T{_iQ>d_CzSFkde-lbC~7!P8Wzko)q`XMOntWJb=Awd#hVI@at z{Lyhko|6~9_PO86Z|abHkbm5{dS5K*FDidm5Pq}v6!cPxRTPD}lf9AV)Pi>?Cb^O) z0`Vy+!B0T@(e-!^hIH})R~vQ*LU7nIfJu!`^E!ohX9}dq+V|l}$B`*`1*uoh;F8^> z%jExf07D!j;BkRc!Ve_LkI?Txs3TmP{>J*4%zhNlUQz;sNxZ{sP!Yl?(rW&)QaogU zP0peJ9GRB1iZ?J<()bQrYD>5XSc-x)Iq$=>11_Mp67!=3CJ5EqTz?fLzkl#Ny@bdF&Q0Fp9X03dGhaqf zvj6H%YNUTcU!#ud5AAD(jn|kmK4SOI3bSUSyKZw=&3{}+r?gdaftdb4;g+2t0j+8Jm{-vcu`a&!CM# zg$$)nl@FuZAR@U4dsTmOp?!v6=+y^tTuOvw9_>`Gwz5A2@+N{?-IBs~4OK~2p%Mz> zKMLB|IMK4{6%&>#TDb?c4P&2{zqgFl`vJXBBe1{Pd2;jhFiaOLquxE)`;I$lN3^Vr z!5n&zZP5LwO!RNwYlFJ_^;4j|a-zk1HEK>)aA_&O_ifEm2*LfebKI(fPI&|4?MGg% zF`-cs+Xo>#>bFwd9OdAc|721cQY;&H54K{@X~*B5Bx4>#cof_-6*APX;4?ZtE*hmk z6d6jpFoYY3=~oOn;wZ4tvk`q33)Npgm!dG&xg&g(L-;A*dF>*0hfS{ZwaffMTU9)vSu{YX z{k5w``+NkautKrqe|ziKXHYUhL0Ew7&K2F$wMig^W*Le6_FUl!LmpIT_griPCI^@% z^1n6Xtz#e>KlTVG$1;4E>&WsO0!~40IFkEH;8#eziiE^>=RzAFGL_y0kuVr;*PQJ!Kk9d^`#B!PhRYE8H*9-~-SE?s>_Odu`171qF=*^78)xNqT zap_ZFy+>xKh=hvFtml0>!|MCH@7Hrb_g~LHUgtQU&o$nAUKgbB<;}|O zTZSqMa8(R&q0^SU*(8hXpNwjHp<>#gj0i|WX~AJ)iLgugrjyxctkk9en>74&Xf4 z-jq!Jb}Ikz`^;b0I)aCer%Rkyfi&Y+KWhK;Hr^J{DU!o50nsYyah_K`%7OERM@!*4 zw;$IT);{-B8}6aG)SR+IYB4?ExB(Wa_g(k3dZz83U|!20&s&&z$LZq7()$^ zdy;;eRYC#5C3MQ26$?JERtTsZ0FK;xZ%7`{h)2tVG1@As2n=rQWe8(@wERO zwC85QO)jx7P^}?I;U|jO;g*1ES5+KuyA4bX;%Bg~y+WmO6+^?hS4}wvGN7=EBig{^ zxIo9Hu!5^+_-Cu#W>qs4pa;D^f9`o}AV`Y^senEOJS~9o zD(SguTu_H-i;2IWl&aDa&4$che=qj7oxZ5ZGiNmV8o3{}GlU#tc%F+k8?Nc~8gj&G zx-st4yRhAL;^`kuFJ#KhAPBH&K+dX7ll!h{<+?4kXe#Xx&-wNtGu_GK6Uo*cjnNkm7Kh*&#=0u1+JneLiyHO8EFIhrj zMFcLU(1k-M-;L7P1s>@dFIclVN`mdQ6R3?rh_TxKw3FVrgAw)VLd?49NWx?jFST}G zu$Nb-!QMU>-Y1u2W)tkrLeZPzOO3Bh4-5{0dPs!(D26`H8`$Pl+yPyi%6^!A5NWgU zudTk9VPCPh4P+i;0XGVu%T{rJU&EW|VvpHDv4QeuMs2^50Y}n^Wd^_tlRgK`v|d5v zRpHuz*svf?&6JBmHg3O2sNr1;xdmc))IJN)47D9xss8D!&))p~w_n3SBdi8tLup05@h_5w zr;D$|R|&7Lg_L0M#e<#pYrT*Aun)E|sF!1Bm79AUF>qe%8&?7I zV!Ifl1w3#rT5B_QdObw;--tUME1<188ti!QV7(0#V}`INv_X4~(BRF%d9+ZYevpDS zzV7Xp7(;p^+&a!eK8kfD+~En-rKW>2lOEheU-S~d9A<^i%O}h`p5JWEhHTAmc;Pc0Y#-gZecMa}(Uj!{AEpS*@(3i&95FYNxf>X_Y zO1?b#r`!1qtP2@pm>J(a7OE0^j~XFUUU2Cr>m|)h_=mz3A=1;o-w%DnmbYEbP1L_y z3w|jyj;$i=qx|ml;-_1?>{GS1PS8=gR9>jX-!Ki-hwBY`M@H-RmmZDI zRTpuaou+hBe8ZQgZUt>^87QeEd`W4Ktkn-yCaul&$MvCmf=d$4Ca{EKd+i}e^4ESl zM45Z9)GLZcA8dQj`4;6XyrT_tl~KHU5CZ9Trnyu)TwCFEg6wPvHp{k-g0MLUmq{G^ zehB8!L(P2ON4qow+8~~^#b7Wup?_-o+=N^aNZDBAaYj5jViY`i0dO{?n2%b>U&CE+ z`T3S=DBBT4`Jp;E^3X{7&ibi0lLCMmO&dlqGflCXxda;h}awfW{R1F37psHnOTTBpqs*gjXwwd@G6WzDjtL$*~ z71hV{00qm5-(R#VX(mCcw!z8$M+cMP^hJB{)$ZzuhCM3#(&{7A3E6=Xu$0*i!cr0_ zRzZ-?l*JJEj z5^qF6HL?uD4;t=G0Jp*en^+*(9!B>99%8 zE`{Rc#3PTyEJ0-AoFQKt@T<8fq6O?)3uv8J5;$sAP%f(tq4!YHs$XEUL!CixNoP!K zm&Qv-zMM^k6C}-?sU7)dsklN7!9R0Q-#!UV1ZLp(rpRq~X~-FmZ*?fy{7N$yuGvY1 ztm=;J6MFOd!}{y7dVYyBTPuZeK@5`i0~e^DcmL=(BgA65G#AlNmpQ=OuFC0mQko>rsRkrk(>C?NKqjx5un7etCJ4NPhNRLc~QjuQ_P{u!QSwnX8XP;e5R>fnS+KZSt*auSf8V zd^%U5&!eeOJ3R=jc$j>mU+0AKU(ihKpOH;lP{LMy{qzOtbJFt z)E`gOgrWcF_ar0{t{riu+Bomk&{AGt6lvdDxK-LYqs25t8=@rFC9gI-ZM1}$<-v&@}~<@5Pd3z0NbwmryH^FkY|t}<3vc9VyRL8(p&erH>w8t z&N)2aaoI(e>}e{GMOOcfm!UTJzFJ*6G+qX5i*!hKN@6K0lVYzZJ@vS`1xm}J_iCjS zAb+O!`e=lQV~L2!8rmYLNyle9=XGPdqmdLw_UUe~F7Xlgb zVAK0`6(_ePJ3j_+pO02Xr~aJEa+U482K5)OZo2vmN$&drO!%67Bwb17^i72Uvj?vE zQLlU`f4Dm?qD;IF<$*&8W)*Eg?#P?1YkG6n3Zh%LrPRi}7Hgi}J3-SPTj%jU;RMhu zgZMzGFo6;1-9bzmF|GARQSCS>teQ~I1^X9pwW=EX1~cdybl9rgd}jQ^;lpoVZ>N!5 z+^UW&0}dauU|PNXCUp;jxEJt`WEd0710Agt*mscR=kb(uB86Q;@6sHsCr61oFejX$ zSR@{}x&^f0nrkRGs|$YHnl^J}wmqY-nMpB~RXuT+s75EZ)+9vT4t*#(XI8PvucRmoi2RNw(7l4Q| zfw54$odZX}>)zo>p#eh*h1VW_Dj9=V;7iuFu|IkL%}5>&_RI8@QxI4dR3L>(w%vtB9n5`HgHYH16d$S0cp88A$G3)nmZR|x9 zg44STEoKX|b8p7Nar`ZiLhTk0f*a;*ph0)wx$%nR6#gBLlK^IGFA6#L8<5!d)^$g* z+Igg4*kM`Sv?2P)l4saQIxQhCrCv!=8%{hYkR5Sq>@e+hkd~HU6ok{YB8L!swQz|Q zjWqa`;$8xf>6@-!5vF_z^FW&}sS~;T3@nK~wFM<{q3<7QIV1?SyG_%^U4b58bp8Qj zv(l1IoB*GKjiR3nQBz}R&7KeoBcO~NLM(kH$hr3cffpWRXWdA489rU!IN$g)zi|g> z^YNFnu)mGCKjlWp9ck;YA23*m_B7iy>6?M~vtcE?QQ#M0%ncjoRWT4t%Dhp@a|^-L9jgmOx))Sc>?%7^iu}0PMmxLLoIAnL`^_OK{RwLVNP@{G@;+ZANHj{l3+Np4dw>y86UTu~H7;%iF-w0Pa#62S z;bff>h-EOo+_aRN*~yT_2(6A>(m|?9;)USXKe@`bZE@+%0(apAfMuz>s?O3p-Ulc9=jv42ci36L-L&e4LE>XLXPeS06KK|F(yS z5|{qn#qc1KKWv5D6C}vHm+6}1NVic%evlyX(k7tx?p2TW&97F94E63S1_>5Y^`e7* z&7s-gv1QJ;$5~JV(26>6!Db$}c)!Hc35Y-rp(ULD^i=6o`bB7M6>WHxu)`wJtrspUTG1z;s_haPpqC{YjfD4n! zuLWin(S`H;>gGqY=V9u}S_Kh@DlK2KG)>N>PqtmJS|6>C0wIUtYjbDJj>pgyC9C|fTWdBq{^E#pw zxrc#`lHa@fFZJcC_8HFj%>*64N1rQy2m)Ri;0zUMrjukFi3;aexfXe29S9MI=gXV>%X9oPljU7`nHt^W0HdcHi|SCKzuB;GzNv#|Mfgc zNG#!(d1xlV?yeSUZO4|;&S%+7;Hu&;+NR6(D^`sR+W68T>LK;7nuAScwh6xK?R17y2Y&ZQ_V25J#v0?T)%j~0tw|r#~g`}IFri?-*a{31vNoA@# zk(Q#})y|=8iVRc>j5r%nl;Dk*Rr5w5oK-28)(iQJW@w(+E^ly^XqYbpZV-z9ee&K* zTY@UM^w!tQ5W!Ew0W+_GG>SbkC+X zxpWX{{0c}cZ2iBnC*SA-xC;}?cm{o%+i1fZ%jE?>2+l=?L zh;ofI5j7Vm=PP3a%s|29wklM$q@-v!I`GQdu#XYD1z9m)*OYu&R51yYtYu`9xEcrP zAl|yj3k`7(^=(RbKOB=cTSf$gufE~1=8@N0ELdl>uUL^O(zf+rX6Q&dwhGG&8&#)Y zp5s2DY6vHHjG>G|j{3`w=roZwI7sPzV#VSn&qWZR$$_~_@eh7|KNyQZ;%iA;a=?BdeIepH0muY(>J* z!lyJqwm-vm7EWJPfRp&QXj|fA9H=8dW#e8-Qm5xoB;*PNvz{@6lC1aEUD33nO zs#!%%T*f?jN}>zUL9>y^y~@7r3(bdo+GR)}WrAS`9R>scJ%x#H#TdyF;fo&%J4c0%R)F&B?$_6^{_1E4KOx|WOX`2f zqzQmjl>wvk5nMA+A${AuE5!7y@&TBd*ECN240#>GDdSeYrL09c%_9I^q*IXm{bN)M zb$_T``A^z@`~MKMZeVdwmft*0Ngo1$GGd|irXT={_c%beaKk|^4~z~Ob&vDpw zNEQT0cHw(6{{iR{6i<@kNMBDbh+M24tRW;jq6J}D!*_5A)eu%INc+9>%^S0H=)k-F zSoXy6l_|7Sj;>hLuK3_7ecyY?>;?5LYEU$1MmW%8GEXZX(XHK)lL3(8DG;am9ZOibc$-s`0+GGyr991X+}o zh$tb2l9wOa3Cs}S4-eaHQ>^+IU*c4Nj?~Id*oH^LnIf_Da$d#+6B3&SrkyI|`x5y@ zQjoaP-AKfD-|<{SvNwqghM-i=KLm6>}$2AKvLE^@3VdLiy6@;??W8+ci;j4yX z{QubaYH~kHVK^uZ8oD2vYz&6*h%!7)y}R0#fN&f-i%e~L_o{j0W$7Pf8klwMIkVV* zA&}ApnaO;IM2>UB!jLv)CwqyHVNh+3A&bu9jS;ER2DZ<2EoC|(A*rcFgO{MHwZ}IsNXl|C*fFaSyRQn<-${cWJHf3%Dw!dP zRnjaAFaB5a$vhYQZ^kedM#Fzn28hLa++0!6F#l@+a$xUOiB`D(?*BA8V8@Q)2~D5L1_ZELgS6^%7mR%HIJ=GEV+ zyp9YCGcq8@mignTWhNZr8u`&?K zn27VzBLfNhxn(bz>^=Z<3B|qlq-mEgf2k;=2J1I=)%SrdSYKGwkt3KL?kw1qKC_a6AzG7=JS^RtZxHLp(V-vPxg{VR9 zW%1oe0>H3>DBn}$UjBTC?`?np|L^Aiu(pcqlei7 zzQ$QsPYOGO_HplU-?E;n zd9BG20KX}p`ggo>5fA(ZmK(Z~LL!}ou;9vF7Z|_-X45{?{gVizQD#!@xJOsQF<`4l zVn0IB!P#hFdL$2?F|FG~wo<^viNV7RqTQha{}>d*j0A%F;cogeBoSLS?d2kh#6l#m zo8|6x{X<@tslvk!-h5{NVi)BO*hHC=vv_VD5?HNW#kB;(9YY0fAkUY#T{R>bNx}to z3wrpSd-2m{R{s13)e>NX9&CBsTc7ZmK7k;`9YlKRS_&MKN%iAlg{7ASeGkSsU>$X*G=OFSo{lB?Xhj2eEOm=VQC&Whjp!N_A9qB*n{) zz(T1CKm0SyNsB6|m;!zhharIlE$IKH?@`Bq0(@S~LNOQN*x+EeU}WYW>_z}dAsh`H zD^`3v@(4Ou=jA5iLa_J)xutSR>`eS+H90SG(${W26#Fl$QMnGrQE;3Pn-E0NmYy{g zeFUk6S*V)k&bYzuXY&NkFnPIka=AT=DS{CMV~mfkBo=@QHOi;rrUzKD$awb~uDM4< zZnqLF>!(E@GggI>6mN{Z@;}5IMPP)qTI0p1qc808{~(~Wn*-M=-F|!Pk;VDJ#iWR_ zrnZ0^G&P`z<*SsKH#L?T!RpJ_Rk1{~@HX-d#hk`GF$WMo_SAl9URc`y^l70$fdiSv z0tyMFSaCPX0c=w4+kzED{ISlxglF-hh;aPJ|43wd4^@%g3cN~^DWonmoDDF5T9uY< z%s=^f#i0Kvt|T%cD+oh9%Kx7mWmvee4W0fU?;Q(440Kl9ie`lW2DBQAQI@gTR=vE6 zL~)gtbrew)FT}%1F>;dHTY;p-Byk>PKRws^tMW(!e$I1j>Hix#rkTLCii%f&L#_uhNqE4O)sGS@NpH98;m^xlg~h z98N4uGuO@fW?Z^uFD(vp170A%7g}#bD?%2sMkH1@uffwaCfCd{M)h{Abi$;jcC8mZN9Lwj0+gD{WbhqO=9 zPF$E9gx~)2gRmTcSv<*fcMT=HeVjYko$!Xh2KassmvbAlnjD;%8k5~yI2K01d67lC z{nCT3rRq(P$!FBp*OxAKh1IQ?&{qk@xjI&uL>U0%NKv4b!c$lgD$1S=s*W^rm3;2WFNW@$S8WafEf$wXsMN z)?tG%`)3`W&vGL4Jp${e7b$$a7$bzWH*kQiSmSq+GD99e`RW~64tkCnW%%WeO|a+k6=QB- z%7J`!j$7^^JIY8?!7LcWLR^d&Elq{FXC(O1%hzymE7*+fWsm+r7InJ-HZhl**JJZx zaW2xV#0}`|fei^>Tu=PIFzR#|Kp8y174XN1AbxKV&E0ngc6{J(f&u}Tw_#`=3y-}k z#`;gpU^uejogB-=cQ$|rfA>ASg!xKhU}%vk)28Kn!SW-b%yAl1S^-UE6D%b8kDHm9 zJ)~+-;>d)NpIdr)5mw^>z$taQ`%%ttf-HLOMErQ{ix)4(pLQ;tz_1zKdLLuUxnkF= zuvyd~LLPN~C0|Dm81XHZCvXhr51=a8?2;*&HskN@C(bWh9Cl>=#CX-nc*!QJ6&Ryb z+X5(yGV;Voz!-J46uAVPQgZ`Y7R2xcVK$P@u6e$KdhDn^!O6BRKMkOsF~XQA<(4cu z>NIegi*za{Sv%~heM>%Z`n^C<<_BJWqck}NPjGsrDXw7sB?F;VW7Y%Cb?#a6kR3&v z-x*j=9L)=MU>*?VK-ql~Ml+aE+qmN86(oeCg%x=z3sYcwK&T2Ae@q}?0ecvxS-0C} zHTDPqn?7yOW6yFKvl50I09HaAKK|sbygBhRV>A9y3HJ(!$&STrfwor; zLimp?1H5yw_hZ;v;7tCjrR?}^V)bA--D!y)JC(CxtoB=bmh;{}#_m=EfLz*7DmTdnU@VAFY)9U#o_lCp#Lyk!hjzCj#>;_2knBTIb zIS~Xulx;JTTuRU@xC9AydVqQqjw%a-^MIEanBEh?ZwkyMtlKZJ4Mv#V=}i>{MhBO4UJ4x^W;mp0*Is&QL)Cz|j5DD8 zAIMw2vbBIT9_92$!QwDzMP|hv0%5!cPD(lR1t^lHFoLo#l{$cq(~Npp#bds?X6Iv?IeseseDQdQIjr!G7cD2^-&SkKj{5A zFsrL4G!L35exsbL+hz}lVX>1Y0!^hVS_MC97W|k-h z2^_SVtc4D^`)u_<;EJ(sU;r@1wiu2)7_)ECD~zxXaip0X+#f-4`%>Vacna&M4lz1O z8hqiyMH)~Xt4nv2w}NJ`-*dg3h#@d~*DO3nB(+2&^D|$L62C8cLijiU@_@JLjv8cw zo5ltE6PBUu4%2Tuv2Vu;5qu#IemN?xczpSa!Cg2Ho7)?iMT6Y|Our$hVk2QHY!MRm zMYT}Jy+9CnDVhmSHeCKjFe8?XX%1{BuPQol8)E>A%V+l{zPuN}Y|fl{oA`ZU)L9U_ z#gQx446-r=mew*5qQCVh?3l?#V>bbp|7jlnCuTUHf%t9cr!0+g;(&+J!*q`b^O+#8 zAjBp8(sC02ADkraG=S#A+~lzgfJ0Q?(J=xcbR$zaJ5ikp`wBdahibo0Yier#I{IcQ z$rYpkwckc$H4H2%*rzGXJ!17VA4uMdfp zzF!a7h(xsr%5gw%s`78PCau1$3 zzU_R-0_y)Y5E>E7h)g${HWl(a$W}Ne@Ps(pw%srqxBMTPk|!`){@>@@{sG2i6JR(& zqHcS7VG~&;GR3`LXcIBw4%2K<=q4k(4Mq}=NgIRW5@KF94j$r605>xXI~P?35@?Cj zj07?20sB<#{`n#dgG9{_k*eif^N-OykbG^ZdLg+UC|&!zpELn~Ik244^STVOxXUos zD=Ni*aL4kMmVo)M^iwAu#|pI49T%iP5CdrdG5iN#`(|K1Hv$Qic(pqZrkn2D*hp6~}Ax39^O>GM^Tcm{X zhmvl|NtOd?k)t6gI|<9c-2|Adr9VopvJ-&dcN_|k=gCExw5^Lk5zl2y51KQGxpr%@ zcFdk-b+B?{b@`<{?84zaAk9>~iCv-p@!*5S?6wGi8+Zk5>RvJ(JTQ4O|KuCF(`?%# zxqP;d9NJ36{x%)~xNYY9cbaXNqn_tb3M&B7obl@+%IFPefST?Z?&E?I4TvxaOK_ck zd{gXh)}Z+0HIB(cX_pL?cvT6jUO@y*S~%m1?@XfC6g-0ti5FD`UKdOe`ps5Flmb6M z=02~kZYK{0eOZ4>-i{FntBQG`))46a${!)Mzt~Di?$}>` zv1psb-5_k^#K|IKlU5m)A1u>F(h-(!jGJt_BGC_T@KURhLF>e33So9@98a z{sRGAt~#%hUhK84!9Zt5mkCGj_l`A{dS#y$O(=ZEaOP)X>Ve7rS^hvlKE~Z{hZg^; zWY`^L&2?0QjlIF`(32LbfNs0B_DaA0{e zo1ef0pk0OLNynUC?A17Cz|q_Vz|QH_md>?EcK!g7^PH|`aABGsG)%R%wEnq0h20E) zfW4|DOl|lG{BsO8xYJ(0QX8;Yst`LsFhPhi21`o(c z8vrvE3Cos3Zo(2l;%iWQ1`8kBlNh*_Zfo_$Z)uG?@#HRu2x(|tt7{Ud6`S_6g#N?I*4*#X?Ct^GH%Ien355r2 z-ZeM;tSurC$Oux8v0Lqr;NF1HaHB0IKmyB%#3p5{Ao?zVE0`T^Q_A1n`#s-%&Dq{v zqz$Yln?#m{+;(*mnfPwUs&JdNmK7ejJUcQ?AH9WuC?3F%;<!*# zO?Hqnpd(o5&&~4(j7-cnYUUCw2JT>hSgk}-2S0$ySAasv6?|f-(bn2y8zZ2|hFBE?5sc_~g7)Knk7HJ)%xZNZ?+E_Aunn zZ;&-?>q;)3-Pm@==E0>!lvt7jaIIoVkRrkL6Sb#-u#oB5mr~iIJcPI}lO8DTE1a4% z7myK}+nq?@8v3)air+q~pwknO%A3Z-dZ*y$6Oj);HAOdcDP_-7Djv6 zHfFlbWHpa{=ugydxJUxvvJaU0wWbEH$34TOqx<@`2n=x|q5!UjicC)za$!m5BqN6t z^p6twf#!lgY&7Ka-jf8HP+pgH?Lx6r>L^9>hapZmp%h(WxB^e}_+{hYzOz%{;_yVE z|1DdIBJXWwmPrIEBu0>T5JSoYb|HCHIBD42pmlY9ye_-z?oF3l9>w}KtPIK%);O{y z@Jd-ip!ICpSx#4iT$BZo%yZ<)$3S&}UxIxVA*70}06}Y#f18@4h8_Vnh0(Mw&+0Xc zWQ0q)z$kA3NqsE+K(B7txI4D+r9-`kPX4QTs=@34uv53#%bG0>&2qTG(kJ_$dr%4= z0h*xoC=BhIc*{9dwPt@&tqa8iDjd|ptppYPN0|eAKxj{4C|{>0&f^h5kk02eqBKF8 z=H;ad;os!wg(<|XQidrvfDdaHc;)f~{p#cg(#cjkNg9ZVJ|h3pA2V##j{Okz?GLp| zq9B3p-?CQ2d74OmW{cVrdM4f0_c1ewidh(~*IAYG z7&U!dk@F(Vv@<42j};7(NWyn&lnc7Chwnf-ydPv4=0(u*+9*KdNHG|Jv4t6fyMlYQ zeYQ(&3Hj-8$a}j)Xs_VMMQULwj@4+G4mvnjHrqguNe(DNq$~H)p9_dZ%1(7J z9&?x&w4IOpLUz|6XXx>nUel-2mqZGawl;o?VZTj+%JL6K=>i?z4l}*E-cxhgh7hmy zB}OUNBaw|W<$Hu*)(IqFUj02MN?wONn8!*cX4^x4x$g5GXo(2h*v7M%&lZzYjWRYB z?DK*0(^lU@_YlIkbtprZAKpxhMPB7)Dq<=H%Lwd{!Su<7To6t2@OZRI2!+Vcdx<`I zkNw(-KGQrcY13%Eak^x2YW9`P`j`q2+0uy3i3TLY(2l+btAH<3LA6I#q_l5>nR<;o!s*EI-;;DT*YZzg(St+_iuBL&z0eC?3`C)hh z)8S6Za8)?+^gf&+Gzu0zPWLkycBu}d>DFH-?*H6D1kwB^nShSXly{O`J5a+>nBJGQ z2Y&sveP(Q8wrAX`9#0wtC@AXAl5-Q23pCQY+e#nLv$)^z|J!clt6*QaeOT#aOAr11 zNiQyK5+&KEAOT3uDG{TEw1KW2X@S>&Bz&b$P?;Y}8O3mK9@NVTLdglWTnmEDTY|P7 z{^~g+Je6_wbNkfUd!S%1&Aihfsz#WaqaN|g<%r6KS(%g#KZTkf=ufL(_qvFXDC5Y5 z$65x%!g>ur8q7&+?7-vNR|Vz{5Gl%n_4%&Gc2W3zxFZ=}QJ`e`aDkby8DI-WC321( zCD~gzYcfs=N#ief_ur%Cl-wrz9GXuN8nF$?^p0T=If6R{lEU)?aTMG^SU{q67B$Q} z%zef1PgV2Z?=Kwq6%`lBPGN0$r?eo37|jOL*%S+gBdu|-lKG!~{Dio)FtQMs<9l}1 zFv04J#DZ@VW)U!f?>qc+<`bE-N`_7A`;DrD6{U#4M^i0%X8s3Y{yoWBG^(B1+>4Al z5g4j9R8JPi1`8il&fY^{4i0dWhM3T13$Cmc>HOo&+LCAWGCOxsZU9RLOi2H7Qh~G& z?ppc9AoC3@h|Ckvv8N2mB>$ZnX4->&>DfywlchZ!H{0el0AXeCxj_I@j; zg!BE{_lU3rFgv4%UPQz%Fqg8t;Skrp=fV~Q!=~gVP)`xdg{d55X4^S25DdvEfH7BP zd#Wf z1t`tPvJ$CK2V`IU&I{#;Le0pgL@j(QeUeMH)yozZjKQfRlJLvgM1U-8YmBPm6&Qvg zGI=g6DT4U_6ZCp;0Olw#;I?-c2)@^if*O!QcP{sALR@sOd1kn%d8W*fei1#afcCST=*iipF>_{NR`*Z9tAlC zO#eo~sp(UK=>{wWbL=7t7FCw2leg{wG;FfFMD#7UtbWVt0~YEda)qa%R<3_Sq+{XY zr7dtN@5$=a?c_ktdO*(G$w?T24|gEwD8w(aXv-x51#vuG#vPevW*4hP2nO>8aPEa( zKIEg!fGlFH{%|8Xn7z|Igq!FLHcs zM*Ka^%P(h2QACXPq^8i~EiRT1%I=ur?`Kkf$E?5jvTipX(3r>UUNk3A#P)}dU;M_zH-kfB)&pAcV9K4!ej@-cj?98E|0by;w#KUw9QX0 znX&pHss`E8WC+kEITsC338X4|E{SW2WJ7{$0R%9|Hnb<23d@Co8Ub} zp+XP+nG_Z>LUyh$dn@#ekA5H}=a$hNrWJ^(u90>MFL*~mfHYWLSVvX~wubrR1P8n% zT21B-dV2#|w=hw69SqoqugzKk)%oRO-U5zDC2lr^94;>xU|ZZgN|3On0f4MYvYGuU zf~c?Rk-%pGiE0A>90|@4IRsRwwCT)mf@Csazo5m*RZ0`~hFu5Wg;&+mliNVj#lOBt zB#!@W=(@wfcSz{R9A4Y_t799B|C^uHH+j;H&J6c17%w+4`2dvRkEcK4DdT@p(uP46 zfZb><;wM>hkD?txyI4 zt^yu1E%1=~tpVD89|u7XKBNpHylsE&Zx+8t!@d(HRusU9vP~^~uvHE`8 zZ6~ndB5{CS2h1W&Q?IswL{jA%kq>ab2nH6p&kzhqQ3gR&r`k8xB85kk?|Ild|F!B4 zc!*RGOB~?_h8^)rjqE|f3Sc%;%xoU635dxPjmLC%!7w}RbQ1}b5fAUFeB2ZQf(K)Y zP+HGRv`+vqz6$WPT2^7qA|OGaBKAwv9z3yFf-7n6wBF(d1_gZLg_Vf&2P3-HGe`%> zlhoS6AQ`Bl;$W4=f}L4|JY@T81%Kz)g&P8hc!?h5#2$xr3oFz~5x|Gtfy6Xz#W(c* z3KY{8yOU@Rb|4L}jegG|7>2NM46!NvBLsMWTve^ee<87i#aEMwqnz?N)knv0hsofy#Xccb zs80C5B`$Cg1!jl-1Rdfu0K@Gst0Z?8NPyI-WJj{(%Pm{>TRCuzu^pmk-?ff13h>5tFfoM|uj_HdC^S_Asy)k9 zr%k4>n2#rYmLtpE%;vG*dAw23`*~MUu~p)$9E(pxdd2w_{B74YHXd%8(H1zb!LKmu zyg(ewDh-rD2M)2)oJfWZIXW4e;n$3*0*u*o7=ELTLw@F_*JPh~=R;GegJe_Qtlk@Q z?Tb5(QW#Cg)gG{_>kbrgb#5EawCv2E7}n2zGH9%qR-D{1J(Onsk5V9VN_r$zUgl|v zBYoCSa(6CrUdpdENR_p-U)MX zy`@$;ZDR9~ar$~U)wDdGkd_GnnqC=dO_ zr)yRxI|5YM8-Fm>(!O!2moq*xY#(gzvN*D`#5!79(zO9BG@(G>Kjf_t|EfRa#77Xn z3yGRFNVDDOQUl)Z*)h>*d%D>DQj@T0n#tq)#zjR_r^OHMld;pT2o-pG?oV)BCfSq^ z>;3B4!A>>(k*&%3hm^uZ@_tO3MgX)>F#}QNERaw)s%MnRQ^OnW-$6h3J^ro8;R0mb z*MFalplsJdkbvhzz3t`UO@>>{G3L|^scos4?Psp z9NRW3#7EXPKKNjNyPea=!P#kwr27o%rs~q219XsFViY0flokO84#)zP5&MJ}FYDT2 zCRAy>PDmwEFL8=Wv&Sp#(*1nR3nzPE;NQz$%GTX8MHq_~%>4s;f8`l2v zsSOKbsM@CRqM){`xJ&c$j7Df&+S}^(-(La`S{pq|uiVa9VVDkirnjZoV7;j5APtQ2^nMLNhz*&cDwkxZc;U%9JY}9j{O$L1$uv{MwW)? zA7?B}*AeI~JJ0vE%PdUQ)SBPHzbv2gZdacBU8C{0YlfzKyoM@@Hd+fXvwXKL3Y{FP z-Zxc%c_Q{i5R3TqN7GsF>}vz}T<&fzoH!Hd@B6XgJ z?rNnDlP#B0HBQ-5_wq4;Sf}W`QeE}g?k^O^?~~ZtkGkYGx8aAqAF4j9ByQ*# z>CqNGuf%`)h(zq}9bpgI*z!)FAHEjpF24H~MW+@^r{h?u2m98rv|X-0rXTQz3!T7e zz=m|9^$1bC4ZBiY#~Ygir|sMrPX#eN@RQk&SDf=aFpT_*WY>7SKSXH8szX(@9UaZg z1PGBbvp>LKe!eDoRdyC%V7+2nOD%JG%#)61>D(X;W1YloJ1Z%BSfw0=4~@6CeJx3i zJkK6`DuVgV)sLCyyVgPe6{HHc4lxrLPbKy0XY_TWE~wFO>Wj;Y`q`SD z;NI4jTv2;=F`7s@`=U=&{zhX_@}o*l!?T>lKE>hg6vn@vx~FnYO4^KQojUa+P{8!m zsZ+CieArl8+^(q{jk|NOG?k9aNV6`xox*KIRFEpKl1cPX%;`WHV>#~xF|!x$;z*mc zm$HwuO3?q>U0o1pVi28wS^>sjW(ngwuJ*}{Z(YdbK(^5wAvZq=hMak9e&&Lr$$$#l zSi43Ie1%9N^3oq=*imkPP>9)g)1P?nfSghoa+I+@i*;G!bI`p@H@LE&hyEg;_Vf*kAl0M6~FBcSJ|TeR;#LeyUH3 z(3TTv@w3Cy#ju_}!pkWsVWg5a&i8nqPkL|V_A)i~Sqaq;Y0-NT30+($Awr$9GcN`w_<-umGvvuSQ7qt6X>RaWpD@v96Nz_n%f?-wO zrKS(r5l*j+O5a@hIGSqmcf_Oj6@`q`RAQ=(!+ND()uXXtEnFtHovlr$lWJvr>!-r& zepR2%=Iv=@+U3AU^<{e0w)#VxWY%azOib6AFwU;B^W|N47|cGrJp4|soz8VX6{E{| zbZ_ovr~KkwSfBl!6A%5Z%=(yFdwg=wI*DI)OLg`V>aFr|bL^wwt<<<9{l9Wmzu|sd8R#$^Zo1C4aj=znj>}

*0ZyCPUBLj~<=9+>|cU?ez*Dc}mjxeyc+~WK93@)n*4CugmLYT(B;*J<2?tm-}XW$k+gX?lFwW zx%)!FYRBLEfA%F@Ex%!0?4Rxyb}N%z;fStFr={>`OKGl?Ry8>nCp{*#hI^verp5}N z{Cbtwt2gdbr+Xvc@lSE?=DQ>Mjt{C0q^H!TemM#*tTF9>a=#B%sq3A; z=A-E{4Kq3Tj6pYo)#-M%^$Rfm0Yv)IG8AtwY9lA(V(&mMCHzMYrXVKk=^><7(fA zzBzUC^yW8{j)g88&i@Ww`#|Z=eb?+Gitjh}uu9nT&;E(rTzpBlgSkv&-D~@*Zl0k( z+h0n=Y#2@$kBE9y?K+pqHt#uLpMiF!Pwe_ZNLEnNENRItNe427CYCH>H!dOSX0oV} zHbWgJ{x&q*c(ng?Rzi|Eo!Krc8*Q4{N!h}YO}dEu!F zGDY6345~#wKJvyz;Zk|MpRvuhpAA{1TE`O)J^DG*Ainj(qmxh6B|Zw-_dlpElupj8 zv;8y^X?sj=x=L~T$#9=DhzavKsRY}#ca#Xb4QEVb_Fs;ZFyM3-?{t|goao@&uc7~9 z>R746^!Hl%6XtU_r1uTx)v?^TLsud0_nN2I$FJ9}y=t(%je9Empt()xAt$9%Y@&yX zJXx84{cNWT86S{;r*i%EsoZ85rE?9S?-l3T zL69bQx?UQPbdIx_v*kMNReg5+ZQ9W6nn&-bdiP50MjiyKSe@twRqClwaVuAT`FEmS zaz6J5r#g`*RM^&*9TqxPF6tt@XLr|(Ofl52yE%1Ar@Cs6VvhF>@;?cOPOV)}J*!;L zJ*x}3e75t|L3Z774XKCIVJBvvk1)5?ItYdy$Sw;pR~QdC!fuc^Ri%(!EF(7e%-<>Q zdfJ`0$)9HO{8#(UK5;eqCT{K0WIl7($(x0tq9w!Wn)6_IZv5YXcS@T3etDcct>0Q7 z8!4|9GZGVV*DrhD3~+qc_aS5HscqkjuiO;xbo|+O{mNjlN4=8#O|iM(?@s#7Sym-n z&ljGk%l-9X&iQLYgS@>)hS2p=tvvV*yPD9F7dcpi$-6FvG5(w$M>teT2r@w~=@vSlBJY_g-m_^}%0?!lFD3DrzT9{ps z5GXC`Iy=ngWT5ls*EPR_YE52pSA|iRp(39T4XHAYYm}p@0C|&t*Nhjkv}LR5eC=kr z;5zi9pW>5T-=MhLP=!C&QEdjUzJp>7hE;sJ9bPi>9f4{*Dw$V{1}0ig?i-|9KdKWm z(EIb~-LG$C#!hUZH8s<86<&SI@PdO^Z}*DzvFVhDvWMqtbj`xDRQl<2gVz${gv)q zRj{fJ3ZK4Q6bs_@eX)0;rbegsLEP~Eq?aO(H~$`c_;cGODz1ufVR6?l@t2I>#nrZF zf1K8LWI5TVC|GJ>v=R$wZ8gMd)t9$ocSYP>U7=iob4Ty|NT8_*Nuj`bmzv#SH{MWf<`EXf@{vBIiT{M&YrqRr2X7imb6+Cvc*Ye%uES!?1 zJhN>&Zcem3*v}$x;-lMO9lINs!x&%XwIQC9{;OJU&saUb{%OGM&DUq^=9L~~iy)iT zdi^{xnomiO%j(``9lTHf0~#m+|_Wva(bczKs=Qk?!wFN*fr|WG*@oYabD=oc-Avf z2rbCGely$_r(k6`Ir2zTQ!iaui+ap?$otnWD=Qv1w{o8_+rxUPSG7Mny$Iu8E1NaY zu4@06%P)I_zD?Ql^Idrq+h{8$>xKAV zo}AeDvw{JJRqKCQcv~G(VgJlGIM`(M@_^}vt`&Y zGN*dJxYBj3oKYAbL8nI1mCrqVq&_z|?-T+r7qg8J^hp6KA(TZd&1wu0ZpLVo}+Rl`tQe^;%i!>zI_V4F9c=-|Gj#@>~Egi55}0cT~rh3)f&$7{V31T zTlK^sVS+JOW%b-d-R2K1w!KYi<(MzG>R3z=dm0Tjm>;bCIQQ&msjf@@Z*Xcq*K*FB zE;RfSl*YolVY+SV*ZpjJix*e+d`)jk^vD7KLP~R6LZi=@flKv2+lQRSos%sRk&&J4 zFZ$b)ZPzXpD}mmc+Q=c6`+pu9%axB^`T4GJ$j2{x%f{Y6VNcH9`rn%ECS#B z;diX2l|z2zF0Kt~0V=$Gu^j!DE_v{uOWR`aM&DcGF`Ik&?WAKqT(K#s{e@ZmzK8f# ziYI$*^_vac)jQtP#d(Jk*TmD9RD|R*YWqeuy_a2=RKAAG{Mxj%o7<)WV+`~aCe02N z`o5_PBj1_#N^$z|p71wRJSQo#lNqxiCU z#jML?-6mjBEGd*Xa}>``eST0rPS_B5kkW3WZuLcc^$PoW;jbo_9=yHsP#<5fce!c7 zY$b^yD8;J&vCv$`Y;mSx!Hz&-Nf-@#vLC<0=Bw@I2%ha+@w8_kdt`a!Ykut5Pds1! z*=_P#Rmp%+XZEMy+*-E@odfs1E18SR9DQV`gnj5k^V{574b*&Q3w{}P4$Cv}U*j`t zT z{cB+#btk-sjJVp7x4kfkxM_B0c@=+f9M2@Em&Ib>{i+YCvA1tJiy$dySi!5&1=+rD z8RmVZtGVLzN!NdtG%0Hv7qj_N@XJ}(RW@8_NL5sG*fU$xUwk3EFek?_LaARx2BSe^CTdN<{JiAB4dLB;M)r|pg#5cpQR`L2FLVV*@AfvrCTE`)sOV)3n_$fYHO z?7;gut#8>tC?s3eI`0}QJ;((f7kC8uLBtQFQ`9Ol8#$k>P$Zz!rq@qU-YHK!V6m>w z(KBckQ}LBS&B~}YgjMghw9HYO&1yDEZz1XU<9eyvL@wj0vlYuuvrCk|C4Jh`)@6J4 zxAu)O+f995H9skKc5Fc%vW9qa^qe{`#Y=>SJP}U4Mw-U@(YZZ!b<1IvmYWTnoU8Mm z21m0RDN|v6o$#i_0oqu6lTyx~fVZppKOS3G~(jdOgPr=M_%N(bA2#%jR;si+f#Ghvpho92LVjTpoadOP+j2|4Q9LK8F(zwRdXYjt+;3*UtKwpqBWX zz7vaU_`PH-6@Cod?Jhssh$3G&s5!*$n7;0NvN$-}cE&3~PB7(XU<0+uc4uUPV#3ZS zWthu775i%c?J~jZsT{!zhgSMenap;`WslGVeEBXbYWg9fVP?qZzWVA+ik7tZ1lif* zokpoQJ}Z>EZ5jP8MBrodm;}jg-)GIhvQPuTuK)-=E8FP;+@$Oa24z zridSlW3#+}=Yjl&{6JrqQBCcMykt!M`u|SlmlE~&FpYX;vxctt ziPQ&%^Xy?-v5fWJLZ3{lYx718MsUEDENQa?iSr>^_0WqKQ0(>rTRf+%Bw}L{j``{R*->loSchtV(};7DJ{VKPo_ zLhJPIcMb^)%A0ee?%b_LeklcF$Z=U#7H-ka762oK!Pt&&CgnnKhUL0npLcWM6Z7=> zvyzPQi^5@_L|wI1PBI4bnweh>=VGy=`<|@8X*Kb&|7CN3lH$)NB7M#*+XD%e8w^CT zZ=H7@M$(c}&U|TG>3h+?y=_C+w`~!mc7krSKFK!K!E}1rXifxQ#yO+Gr03VquTGjP zv0D8u3!52c{gcBchHF2@bt=FsdNseOOMtf|tj=NitNd(H#z0GPD9$_Tmb0$!%4-XM zp9PRe6D)0ip^>3~J;lqxU_@iQ_n!L3SiGjC&Zuh*>trXT={*{(pv6Kmtc1cHf>I)MNWoE<#`Bda z&*ITZ&2i%T`+|A#urkninE)Geu(=X8e<*Tw?!(GqSdEO)?@_CpHYU@Z(n-#>App&{^6>HUR5BvAZV*}j zUD~xvG;U$wEgQ=GRE_-gZZ|=uoX*?wl0v0t)ueFXFcyY6gV34pRt`i-QX%`Mfk#OB zx!%y2R4OtVJdhyeP6*Lrcbd^J2}^pD_Dm|m%DA~bEn5KCCMiIp-i8*(qQZxST)l8@ zur-@yXioJIeuYV&nR5BYc5zLbZ?kqos%`4Z40Wa8TA0sqG zq-({f)ANCeri`;jhu-TzfKFyXpY}C{OpW?7lfttuYm0+~Tsw7@&R2!mFTZBG%Q!5b zx~6gU?e(ky3VZyb3>_)Q;TdiR{H4updBEs)gf}UEyF?-;^IRMUrWOV#v}*G44c1d> zf?4h|j&r0IGiggDGJKZ<0$6>1YEHa=JBjn&jh{LUxjgkDIqynC+VBoz6G8tHk=TGs zIVwIAj+Dj7i{HWP&}$$B43}z*cd-Z=_!4&yYk(uWlup#1ebtg-Sy4JBw*96dxW9Bl zDMzO9F_)B44*~cqC}nWAOAncNojpY@!^6>1Lc@{jw=>sqfzV{54m4?VrC*wD>due$ znKfs_p2CbBQ%tTvBTK^si!%?``ark~X+HkcEK48Oe)g+)-x#yujc*M*JSy@t$mG(` zp!wCB;)3}u1@?y{8a4Q~MG+d3dIJrPqb3928!u=XIL?PH3?{}39%T%?bAJ>lHr&;C z*RuV40luSCfGD3~%R^u8hdGn6-aDL&yvIfcguB&O@Bd2GeJ}m)N=cYazn4S5E$xU@ zMBo#%GZ1Hf@lay)o8>-MU3sCSBL#U@2Agw&D{?Ixi-GNzZX{O};Em6;tH0=uaEUX7 zfN1!f1LfMSI+KxHhq@%Q?xPKF>R28EiY>+CSU#(s@1(rhxPN8wEFF0mpN?9Oc_*RP zpgUg6soE=bTWj1!ryPz`Tvi^TRCTlq<1Q}BX#!ZbR(J7BW|F#VLS~~uo}`~&8c)7q zLkbxL*kZG@{+=*a%6XxJv?ugQm^v6Q!<^n9OlI13ZFtytoOc4I& z$y!BDEo}*M?oV1)nGR9W2P~|~d5DYcDvby`cYX>5RXjaPB#SnDXQ78>?n_sDwK2C= zm-lLHQW_m;8vEU@S)u?}@mP+A&3QJCuLB9q_QqRDI+0$UeYS<8pEoLh2OL%&)b#_h zbl&M}{p8&4o4&;7kE5O{wT}@v8gImL%WY%8XIi;B+qAD#dO(Lh@3Ub+;C{$!Zpl9L zFJX>+egY@JaPj)C#fGLO(1^aYnJsCf0D^dLx;)lRNF@)7F;_oc6V~cGvm1aRrR=rS z*JLs^B%c>`g{sKaD--kERB2h+jM_4n%{uP*b}aYxrRS!qWS{4=TF%uP+>f`J?>Xbw zo>Qc3UDc`@Urr+zVPRg7IYQ*Wr=ic;c1aT}diB)J)7|DeBu$!`E z{ucA6M{wv1jr*O9;Zm?2PRF$#*|pZ1P!dD7ZwIP^OGkAR!F+^F!3&WTcARRC@jQ%8 zh|z2ZMKr5}nXG>D<=9DY28mE6a#sF>kXsK@I&Gw^F2qyNaq}8&K3N^`G``NY-PM*d zM@f;FGxhf9nQ#(*r=J^u`_Fy=%o|ii>PV6L3)*ThoiB2PY^bKOpWOXRT{UJKFB*aZ znT%V;RwwEx8AxaOf7QnM%*FFNe%>)n&p0th%F#odL7hFDq6qnOMofQCcXxZ=p<0%_ z)gG*UbssrbsY6gn5~pj3!r9B*>4fx_kezFObQCE$Qt(kPwQi0-QJ?ICE||rn0J$@7YuTnW#0U*g-mVI_B0Z6+qitv zN$Y7Z7mHcdUQef@wDIe_d`0t9d=IlIHrp#Hp6hU=GURXW7tbG%)`=vIl$x-c{a(U1 zO5;OLq^@LTVd+iWs8i794!|^4pfPP{I%lxs`Jk3~1hbR&W@%viF*_56!VO8sg;$;f zwokdlU$l_aUQykk46swP*O4CAiIf_z6IEaR;rXy@bku|t@g1eq|GOff8sNzNKB{hP8c!K$^s@)w zD&zir3_A}ig)@@Rf`vRZ+h$7f)%(hcspQFUb7h9hAfvS8a)h!(b z4)UF=4C)Id3OrW&owh0aeQ-w#!U}$lNICx!XuZU5SHEV~+B6JdW69TjITjE|{ko!V zmmrq2y}cX-hv3^d4bRM$b30h?Y~d6-hcg=qZ!QW2eXTuw%HF4Kdtq#DaITa)H0F_M zXM*_PYrSUuJVnNMYHA{k4ftQrDuG$wha#i8lE z*4_<;&qk8&s)otJ%~S> z&+RKa-rMmiW?<*~?BmajcJ7kbBJaCSfB`U&dxvzeYRROssQBo8RmbY4)v5)CWG!&o zCp@nb!T|So=Rj2AJz(7}&_W2m1rYxGA|ZC{LDU}L*k*ae%<*5!qHqbuP>ybLtkYLwT%KAk0J)=S@l!Q0e{Rp(ydAh7$Kr!-a)& zU2kk^O3xL}UOOPf=QNxJS=B>*Wo7GI70c6eOOSgo-bdP?ayChlJ0s2@Nwwtzv}c>X zrCwZYzZ#pl4T(RRgT=(rgRk)LLH*=0qt_%MMek&pVuc2~6=?Ubdq+{&i%i zj@!hrn*{QVuZ8Ll7d)M_2vU(_SJMOi;qL{F`w&7v$?hYc-|iN_BV5r1xZ?ky5(&}V z=FU{_p@No2MLTnpo@%F0zs!5=dioQUT+546dM~`I?MD<8Cy!_f-HY`M{=++~T!t3fT zSD;PE`$uL@jP)gCvfRi7T1?c@SiL2?^~n|j#&xGHB2bjUXs7m+tj#`GSL#0^x`JH^qHND4|QC0@~J~L ze64?ttn`1&jNabV7hyNNC{Jb)7Bh2rgi~|iBKBx{9@N44@P%-ye~!&3u!R%x-PF$? z#P*_aHGr?FRiF2`g+c3&sEZ^fL@aVRj>$#!IRF|J@CZBM@D)MfL{);EBo@Yu-M z7T;Tr$`7U)?NxQsVba3Yw>Pv8iMI>-y-e4t{zk)p?bp@dn4D0R)3tV^8zx#NjXQR| z^u`X}nFf|GV~qdrP7HTsQ(3p6uaMbl?tBQRi?3nlr^1WbnX~dKckf9Wv?@tn|B!Hk zLexR?jY)E6f#N}_b;iL~qD-#5vF$s|5rac_(%iZ18z|+|{M(Dyf(!=Bd`*zLvtE%+k6C5abD9X*G{D(PX zl#T7-4)0CNnoUc#5tU2-P9vlho_u10tcjtsh zs4b?b=L)7zK@*UJD9zx^>gQbk>aTVdMzS}n310_NRFW|mO>mgDhj{Avh4IBIa7N2z zISr-qM^FzRIl@yAP?$uXZ&UL+y823neClg0QjQLlDV=Sbw09D2tbN6A$du9|!oGig zE<>-rU2L-cF;>=sPnl8u$jX^Bld4I>o5Sw|VhaulFSMrw=oAH?&p)~nywaG+Mnq

uETA!nTGhKq+c&m3_a<0z4)!=si`Q|}~j2G|Z%FTykZ}@DzS4>edw=5IB zuM_E{(x-S)y`gB*&HRQ2bn;52I>456J0zO_fGjPJw#qRaXm^8)T5U2WpL_`YbKf03 zZQOb8>Ew%j*F(Lw4Z8^!e=_qS{&NdH0v&V4OIeIC-)^J+Ig9hmxi8_v;kA9tj=TLM zaMR#nzf#FBi0pypQCA=Xn0sql=)r3r<(z6?-u&46_ddRpYfUWjP!ID}seFIVH(HbZ ze(dF``}s<2@j>?Hec7t{kNWc1(w%G0blzw6?YEhFZIo{<790I(O8V3N#z?Q|Koup5 zyOqR|G#pkPRWqeGJAZA=MjOa0;K-yJx=pF)ddZ~xnD(2uA?(#ry%!ik?y!7&EnUk; zC+7Zrfr_6mUxdefijMRu<)?G*zPMx3E!UZ8lz-w$SFu=NbgoR`{*`WZlIa}vPm{@q zELA4G+C<6(TfGtq$1^6al()J~=9@j1=DU}T<~QoYDHItmx%uc$03d)aXA z(zG+s^iX?b{soMj`{4;)NBL#OPwh2w^nux;)c6km?v3*R^#L4deK>{QvJJx-YKY**40A~Sol1t%{k!?LgrCF$ zX+_JU_0Y>={%EQw zU+r!aZ#xjt0UY+m*cL^uCLp~XV`3lTh@rNBEl(`}sDMW_V6Jj=q@w8z(8K2$sXYs5 z&I4Co(!Z8YlDn3mn$Nyt_1Tv1OfAXprTF&ZO9ZdOBWPe8z2nWFJFU33^QfVLGyp#X zJ5>QSG`XkYq1di{KayY%)a}N}-0p)Qxk5-ugxwn$MJJGdRi7^%^uluEL_D!D!D0go zP8G3Fi?ros`k}VV>}&4Fuis9TNw`17dFZdUDc_!csqi7y8m2hi!iAJNf%TZufyNuUw)ELPKIAgbiwbs?&uwNa8?s z%ZIDO%B`R3itfjXHD&5acQaZ4z7Lk-ThNz*@)ug&*+$vjvu_KmVTkG{CcmH(oYixo?(NN~_ZuJ@2) zcgzL0V|7q04c8W03sxQ#S{L~J6x6mh-~dSw1{6Tq2gN@00JqP$uM<>dt2vU4LX9E! zD%Q9+FZWM#EYt8Q)t&rlVs0#*4&8@o&}ZcmVf*al0`LeR z)tPxLfazOxZ;)h{qXfR}2M(d2YB~TH5(vU)CKby#M6+LI-u0ZbihXB)$znN>PW}z` zowvaohJU@E;G&yW&6}!`y3o3Fgxkf_=??Y!7LmK1Ma_ZAY1#*R@UgSs0fDe;QcF(o z4Yi@mHdn#!3B2Yt8t9e1xnywRvjpJfV0xZr)TCFF`mS~{?fs(Rz1fN!`YpE;Ws|i3 zY%OMzH%@fqStY^g>R;KzR2SN+`+SuD><;*{pvLb~ofDQ=zM*FEJ&G1II6XjXW^@&6 zzuDPU1TN%x{GJq1JZf;Hnf*h82s0RgPT>vF&h^&1PU&lv?=|*gr-H8gvJd(Y`cHS~ zgkI>$nCIwRSV$Y;BmHk%`MX$2v{Bq@1|5XY;gPUDi^7pci-MgEZorrM;ToZxPM2uH z)E!l829=HPphqwDu0cH`K>e&QJv_q0z#lexT!c6$TWYBkf(1{ z%KkUcz1!pd-A`d$3qWzVLA2C4vH!*KxY1vroBE@%QGF<@{1IWcVU;S06o$> z_d^}966Yl;kaMot2tDqj3qXyeY^i@Zmj`%maPN6lM9xgmocmOnkPd46D{$-pZKi=? zs1@}6Hrn``+X!Ubb~cK4GVqK>y<@YTc7@QlKv*~xQ431!&X^-nZqwFtcyA7@dykjd zt}cLnWUoN_`B9A@M5~ROVFzF1Y)1X@G@&OQP2ACjOc1}=9mIu}jnBWFXGVR7so=~= zi}UF(C3cDL1w7^G!BAWr3D~?ZXP*4r~}}-0klVa26YnXg$%Hfwh5x@n0sy@mCzFSKn42*G|)>(iHSa2!3@s)65J`@ z)PsGJTMsgv!r5$Yeo29K$k7b@t^09pp$%E(h1+?E3^bw3ERFWjdP{i3zkwlU)gUR$ z;{3;F5*Y4y2K)8w>cwX44t;?(ot2MEsQ2wfomD>`c)+9Q=(; z1-@K>3n^|i;h}YBLF=w;8utPtt>JOM)fVO{CxFCHxd}Zw0^bB~!K&YSbS~r|+y~2d z2JSp_jQs>IY9Q-+4YzEUG2B)W?KLQ|pw{nS#NjG6{U(eG5x5H@!LiaJ-c=~*p>Xn; zh1kgfCq(;hDrnUMSy!-6qO#e?rVrRbt7c@z1K0tF41k51MEP^f>){_)#2^Jr?>P{O z(!=M0^rqTVh&gg#NVaMG8Gle;De47Vn^f6>c1jms^SsTT8HsDyMNrnq8}{xiK@`%H z_2AL&;|a`Q=qJ=Q;*gqIMY~R;kca6;iHo|r3S2wn{znuLm?DLI;7mLQD6IlJR?4eg z$G)A40Y)}oT7Tz+DSF6>Q#4q~0!!Ej6PoLLO@Ri*iB_$QmPG_JsK;D9`+rY!&%Os} zXl;QHd*Dw3i)b56?u4AcZUc^m?Lh+ER2b2ZRJ;qjaK!By!eFcWSa{)SUua+=d?iiS z&{(XZ%or1IB>u&mxDuSk6SHl1oOr(ByV8xLB@pwZ};nt6C;wzgkj!h zUdIs47!2pF5JE^i|iN@r$cn)UK+`yf1>cg-G)Fq2V-S+7=$k79ow5`s6Yb>v>z=L;A8;P~j7+VPXzMUA#bDjmpy6 zyL?kZ7O@xSF|yu18y()>(1;2o~eA5anv8{;zF_?dVBf|Frwd5aVT>u?+2Me$8A`xkZ)T*bfwlC(1 z7zzGIV3r~*mrPgr87YM^q^bJeGGD+-3JhNpsKAIyR7r(@H{m`3Pd+twK-(Uk<|+jf zaeN$$X({X`0Jk4*F=r&k-i{_q1i!0jn9-uixTc|thhr_UKhFaup<$TnIlUK|Gh|F= zC;!a25Jb=HopH3*&ybglGnv`q7GhA01pl)#1adI62dZ2Z2*ur!0_Ij=VS@YK~_j26K`w#K_R?j`>Svd;)`?|UJQk#UX) zTGBCuV_gt0iQvt44@To~InhI&JrCI3**$w_ff#H}#Qb2w0%RbDzCH*HL1M=h<^-hh z`?~*#`2UFb|F@9YhkG#|udgYyGpvln;ZnU+1b)$`?88VUBu$_nQ}7_^HFCrtuaL)b zd|1mIk^R0M0CiO6onQctxM4mo2dyaktNQ5UL3lY-cc=!J8h_GQ4!B4eOXth#ggV zQabB8`US?Rr&`gQR+8I%6QOi^r1q{s+K+;_7TlS*=>VOm(vCkw&}jf!#%hywh8S!# z0Nw~1ipkomWhj0SHd*lj^Jn0Ozg<$_5R&70XCaO-kXsmHA}+fw((Q(wO+SfH_G{A{ zokjPGot+>=?YB75Do!i&;iVA=^1AOOBxcd}4s+RF%WlM@;8B>#o22jlWM~5&)@h)I zJ6m;obD^O|R$f($`^|KHIg$F+oz+VI+8X$c9!?^B1>uIbl^wt2f8fI*JXh6=dJ3u( zb;2*r7Vi}0HXH=+Fleix)(3{b%K`-mvKbVHV7(VS=-b2a5G_sw@hy6Z*>*ANzubj( z02BibSsFh_$P}8(K&b|el!G7~*!!H6-Ka~Z&u)~QYwBa!Nl#mF3*h1!cBTtWv?N?m z&M{`wX^s*+w6@qRRkvlxQM9w|pw}FC=Ti+s-q`sVeaTN8>74HZo#awg^wc^(Jrr3R z3Xp=$*hJvIn91fBU|wIB>_tqx?uR6N;by4E`rN6Gf#@myDCW(S3LggY=4*F@Iym#l z+MX5vvIlL$T?CMa)0uQJz%&RhRe8K-o7Z1LMxM9*;;X#6ldL5FmL;Xm&32fYfm?tS zpJJRuh-P25QkH(Yaqsm6nFN2ktD!zXJ_<7h3-y{o>_BSHYr{dBmAw?JK$fva-PzeS z9>>m}ciLFaB7az^4hyus-jrVqZ`)joMRB`5dq?3*n6eo)c6}cp2ke_b%`h|M=&qF2 zsoxd01h{aso5I0^C~4sd%vf^ljrU#*D1U^g@F;5}z3B0gG5pwmhgse$(sAB{-1@DD zQ%sp0mJLIg9(r6=&CBe_T4y*FeB|lFSv+BNFobB&tU&ZHJ6{zP~ zkmzR0;U*c_7>n_{ueMx<&;L4aU;BQ19;*EZk9}O)8T8{9Lmv0a$LSe{oy@sFl#eQX z$tzy^at`30<#V*X>k+iX3CC>RseHN@8DOqL8n~EQcz`7U=a*u%!XFP+G;$?hH>i@p z{25s15iH2fY>i_mu7iubxvp1n$}nczt__%aUF!IVt||3eWt{Q#o+79Gdsdpj@975Z z^xd&_p&Ue8a}I0s!$y^zH}<%h9?`umEO@@nx(tuw+qYL2Qan|Rh|Zj;-g}6Qij>{( zwq5V{D8XT0?wQ+b5zX!pB?^Mh>2Kp9h;>=(-FKdI4Wxd&%fXA|&6Vs&@Wy36+%2)b zYRC-~maXaCPWy7RIMBLYLEkA9lcum2#~gX8by|9`E~|g?g1%(I+}j_i`~icuFzrfG z5qsp?96&c2)73kI^l-C9gjt{GKwc|aJN1I0=q0^s-sugqE$Hkp6c=hN7FIzA1fcTc z%YOXV>dtHBVM0lADUa&IxJHV)eL`4t#PY4Do7CD8q-jDJAKdI(5x0K`5!d?=neK!D zoSpv4Oz&`@yc*Mrd-e@R8eO_lTY{IG=d{7OMu^#@HI&H0U^j!Y%3WPu5Upm!>fnfh zDIcHtSmlp{{t=RS;+*tIENUXw8o}Swj9jsr$iC$~IxWvAqO%Ho>m}FlF&je9HU5{x zlRZ0x_qY)pT^wr^6!xPq3M~P*^R1+)N6iP1lU-ivb1>-=U-CQ*!mBE8zAb;Yh;B5( zAvMK?+$G1j*Rb+&rXVBD?W7U~g4LWogci0ZF+}qiz3?(!2`83>ej#~a^JejhYdZ|S z;7lw(mgE3)MZqq3-VMW8^csNM9}a3@0ufZvaOLBSkIn}zEZ^)BK^OJrQoFi1EFLCM z36xZ@++|RH^ID9skN;#^F<;VQtErBAd>d0N3teT$xh1q4B8+AxD`+Ae&)ci9Pj?xl z1?oiDRTN}8uFa=gPb(#W9btD>mlWQVect=fUfSQ@LmUxcXVbRFudpWg4~SEdrCl69 z^tVZkvHkz$9Hv4+iIcckH1-OKi!iMv?qC+Pfm>WlEz(A(oPU)Zid8Rm#zdk8I1hkC zzGyYk@+zF$=IyDmuB;?oEs`9g_EgpT`PN2_E^%*k8~8SokDX<{b%sGLjPoL7N_ie5 zzuwYglhIgUXQBAnG|H}usgmr1y3ZriR?gV*btqf4i|U)BKnJz#s0q==Kro2@(ZKSE49a?9K%Y`Vz!#C$-~vY*2Ml zT9G~`sQg6!$LgS85JT~1Q<$ssNPDv41#mu(Fwu|(&`7k{!H>;(GI1nT`G2!1I6 zRkcSi{t1S2!jcaXcr{~)&JgIwkPI7xyoIt7OwY8aK#0|TdVBRE3)9HJ_$Wue|yWEpG@K5jhb*VTpG=0V! z1b^kIve3 zPXGG{Wgfyp1bD2Odd(HYA|cFLCxf(>*5MZ~kERgCSm0C2X(X;Z!a-$Eq6RL8tt`eSIvIuGhN5ZP4Y5?&na&$ z)x%89e*DsfC(=2_s-5{ZX={D_+jIDhI*oUlm)dnMH>Rgdv1nDF9^{tbw^`RI_adyY zadqB)%Wg!wzGl!E{q%wP54S_q#-poNwZTlnk z-SNPD=#_|b22W0dvjbQM_zw*C;5eh)rRnhTtAhmH5`HC~e zE5BecG&bHeya1hj0n{|RtPK-cP>)8s6csGBDUTF2h3&f$!ZzRzekD{Pf+*r`R=&A$MMWBc2&_tUX66w zw$tR8o&@>gP2j~AHV2ZZMtyspXpjU?rx)U26&bNR64bKIKRo4bn{^d!%y2)fW4n4+ zK|x_a%ag+X1JgmLne@YZV)NV%eqUdiWtc5k@QoQY-q~8=s4$&O$?H0zAos+k-mK;sb+Zpzj-PS3$p7Fz{6M@GhM9y@2J7(~?H64XNqkAG`iYFyIHiH9EO#`7aM&I@!}n}4p-vG7e?TaJ7(rddBL^QNsEgdwAKttu!O+z%=Afis;0}(F zRe^+?2T7K%?p*SLmAy@2H*})&;{$ht11Vl+$dJ31({n#L&nuullh((BEX;$&f1~9~ zY2ivAUAv0Koq1*7mDYRvk97Lppu^?e*<6S&Ih~WTE3X&QQ&sb;0e=|GTB*-MN?yOwdK*=P)ZIP1vsEId+n3FG`}_7Dz}2;FZt=?-h@ukUxBj(;C_j&Z4t z_EHR4LZKI}F%>?dDx>n;A=|!g^VLmI?nCacEcZ6~rGMK?Y`o?Z9FA zNM5VywvdcI5kKhDfQ6? zviI{>2VM*VmTRzm>v8(J_0-sBTCu!(TKUwgE;}3jLbK!9dIa$nqxJKhI+3}BnH_5+ z+Q_Q$B41f|bxuvoo1j(BQF<`aB+igUE}j$fGv{ZSXQ+umc|xwm=-Uk?rrUIP6}y{a z#bfWq2)7C9<*xZOI{@M(ka-q(5m2SsYX96>&GMN`bzH9)1zCI?WTD&~5w_xl8o8 z&f;*8y5y9$W2nMc6pSxkx8i%=WtoFnL34WfP%4nefbl+D} zDk(!TGW29zZFel-Txl@@dZeQXenBpqug+^rtS?VM1A%4jE5x|_%Hh-L3qLVt3i{`t zd_G|J7Wna&kx&ReqB;Rupuq&0d5hWx4PUvM?^Nzt;ijdBnVr@xC!;dEO$z2b=E_#U zrT2CeT+iR?IQp-Ge)AhEekaksSUIN@CA|MsTU^QMtvMIKYssF{#s-0}fZn>6`}E?& zsTm3v{ZHVe7rx5O02kkdmr2;1HQklPQYs=sX7(gmy;zO) zSlo<~-TCH%Eaq2ow?AcrUaofCyMG8IJE7BdQ|{x(S~$&QX+FzJaFcx;du{ZqaaX%c z6e(WzMS0Te2v45NvS{Fyukbz_klp{46wD(6)ym#{8=B))z?Tf5rTr>IX~l2W(7b_+@o_cmH;3Sl9cKbKlzU7OS`B+!x@^)s{PE zvnVaatA%5cT$$3nvpLG|^RD+Tn?bUtPeYhT2y4q&=7FCUL1eX!ORdGDtc!1SN$Q(5 zg0@u?`>(7!ZG(wJ62*e|!Fsaoni*{xA4yNB51FTLlOj$zw#} zSFjqyRe%u@)kAV(OqMe{5X%E)rV5N{_|gY}q+!cuZbh1g@3w}l%jxh<_wlL#8gWSM zC%+?1ck$xRuccQa6t9#`3R@eO?EU4Yhj}V zY?@UtdfUyruxFarx5}#bqFSGYWl|loNFHqe;+~qdt-%}0{;%A~myhq{o6A9&sqn1-0 zpA>%3$hv@=#vhEmYRHJK&B+?@@yG}}=)gJ@P525q4wj=x!Alqj2)@JDt}w>!6Iec- zd94i}DLXa=8Tos&xKdwar97?dEzrzQPp%Ybax)TxHB6N z&hkdP|ExR)no-?70$UaPW$-@M+p-VkSG+q~_L1N+0KKw@e>`79$ZrwxBNsWE3C!W1 zbLs*f7_SmGmm@ALYzU4cX^7wz*75^ODlHu#Z#Nl8GMTAE6WOP(f}$|ox{;qX?&mM! zOQy#wGmz~o;(ihgw}fex6z!;zP)j2~xzYTkIoTkFYjJZG{J1*#0O1+Q@+xKOo&#+{ zH`g>Y$(sDDP1)qR++nlkOigbO*)#Mix@vK5p5-ryUL151(|>?FqC4!D)9>8f=-5=Z zva19^^IM(mZ7=F&b(&#Q&`2SRX*XykLRVBTx`vqvW@*-#vGT>pw|t*1fTXlbNf9~RPaZLZZ% zLy{n3MW%!dFIw;A_dBbi#r;bb`glyViOvlZD@LN%E+hOvmVy)4>Xv6Ye!oe^mfsTq zx%soY6bx`+Mz+|uvQR7ve}Yo|F9-SDsJsOqB+k6m!|t-Hz%Rs}CU=NYUgRIRhe;{B ztAr?kaB*gRNJA2YkvC;AcLMWE;_~W1y5Wu4;vitSdzsZ=H`f+UnS`V%9e4z1WJSx~ z3CpYx=O%5{J2|R9{3KR92svk8at^I70saoS;K$Qsy}j%GOefEnDVVDuHLto96zrKO zmqJ^-y>1{EcqGDcUNolZMl?%!4b*h8mM-Swq%iZ*(Oi zl*8lMo>^=ywRbw|Dg z26u{NhV^t;W<&l#8P?jEsEAuNGMC1{x6QuK^|^)B4gyqDXB=8Jq(j+_3@Z=ek;oVp z8qr*w)_s`@L9ghjJi{(`e7?&F$V}f)c^}h%K=N%ngcW&aug}Vqgn|3wh4biVqxEB! zW8X`xL*C+(aelYHYDaTBG9OslNqD3-Bw#5ch(~?75T%=hU*g4Nd%Z@1pz+Pm*h(`u zDik{yNMxkO*@luHPa2dEkBOXogT&Y{o)Qa`u0}NkTz|93^%y1kbvN6Ui>nHuv0ko) zS{5I%6UO_lK{W0Zc~Dm2=6p>w;fZu3KWLn&+v;|!+oX8Qo0b?xEJc=HsDD|oOrULf zCVy589LXu3j@^>mZq{4tOG5kkV)dNX7f)=txkT$}Xbx8RoTXbC%LQ1admy$oN&ZFX z5vk{E%IX1hZ^ss*++In@`XslV6K!V>jqBGU8Ovm+ajU0tn(;4Qv7*n#ZC0%7u!aCgWiFu(Mo6xL zC#zCj-5!IO1*j-tz2~Fqc~6)^oToD6lnypg4;U)~!K1L($5Grz0p+EaGqZ@IEn0?K z9xvdoo~Ugks1hPlz1Xg56V7VV*R%9Udh4UQ%N~bs=R7~p(3zD~Qo39PhrZ+#Nm_Hf z6bEf*$MJ0)ou8#o%PaamJ`jZSHT^Sn8_&01ilAtAxC+XBN}6tF%vQWP!fxbjE(A1; zR2?D9qdf6>hven1!{*Q|INy36Ldh5LJC};&tkz*`YbPytLQRU8Rx&zQ!Sy;68>Hs1 zUH4WLM6J&RTgYR+^^mnu{vEx(M72VDjJKOV6feb>#jt~M;XYozVOM5V&&oqVx(G7c zA8zS3vuYU|#hb$n^WZuQ-DvXx=aka+_DXT6r&xOX65rg{Q*{eg7o>lUzLu7J1>i1p z)PVRARP&8E)3bxNyRx3F)Sc15M+c6rObW65X)8t=+=hHfcCKIxsO^5Pq}A9 zte_m5DYJDDWFedg;9`UDSb|6WCms}lU)X2rfZ+{kuxCU|c1tZ@q3CN!?4$E1P&lPT zNJ|VW4XFr@gJ%1wKqHJI-8jf*#`xNiq+CT63HYs3>k>fOu`g@r^J~p?kP&mwj@qmQ zg-gI=go-PukLNnyIetp_5JbNNK_lj)?F-|wcn|eU&s7JP0Ko@AysyK$Hm@0?cNd;L zQB!(#}GM9J^e((`NzR=0?_ec${K=uVI8hg5_*q zulv_N4Ed7^q_|@6uW?xUtgJ(1tDd4#ZmWLF#Goo(94&7~mK%m}HtM|E?U(1yA z1nG<~zTFns28&B#YGg>(owV)lO^|1|+@)jn684dUs!bXj?P;(ci}LhZ5}71P%$QN0 z4JW%f3Bog_>E)72Z(D;1m`OsTuXM*#J0UWwOZcYWWrx=+Q8z0XJe93I4due$A#7yZ z*9u0eMVq2d73kK;CI2xU;iPky-flZB0t~+dY7x6_8QOIujnx{NKMr4XiJuPp#A@sQ z=oV&AKB36+3oUJb%r_?FLd-azlNW__UE;9j$KM-w2{!JY(w!0HOZ=x)2Y%fxDCS$M zuYxY%{O_cSMSfH(n=l|vHBjGdodtFou;3LC zixHxGsSxUM28CjHA9c4;_EK22+)IC0dA!@K7oy0%Cv+Pr*K!~WpkpJJ>)j5|jh*47 ztX}JGqwH#$wv{kr1^|X1`ZRf6AbM>MAVC?Qu=}_@p1C&-@xcqN_5WBY#@|Zc`zv`s1H7obn*pz5q|j#!rqMe z*B}evUaWAEq+&N@f?Qnd08clKYM$k%2suOLBz3(@uWmJe49Xg~H?Rsma`c{MP}Jg( z_mb=>mQ|?JK(V$q8tLO2y>vF+dL8KVg@#XD>DdpfxTWJHn4G^BgLWRu zsoer`Rt3o78j*^ro9@!nCiX&k0I-I=kQkwkiPJ$!1Tx^D#5L?PneV^C<(fXR zE_!s4avI^HcyY-gkX7O z{tzhk>P9cFObrANM)DYGk&5w!AyPu#FqQX# zPv~=pPR2xwn^RFYFOh_C0_7RS>t22*70H+V(Ab9cY80Vp^7Kfk$(eBZxHC%{z8>Eo z#IAMWS7~=!lzTW-h}V=pJxU&?m?xd-RWC{tE|VuorsA|d9$$kJ<02AT4KvqKB)0b9 zdf}%>RVOWWz=#eMp|Jbf(~ud}8kwndCmK}R9Z2s8?_i_tx+ou!H|Wd#XzY8`2$v-L zqT+Xhf+Is0XF{-4YK4oTQf|l@lM>-nJ3R8_VL0(6Bs%_8u0+5Ag2YVOY=o2=K0Hvm zR)Hzv0uB5MiTs4&N6bj5*MsaC@_tK-fmlOuNQ)%JT!8Vk5Ti)`RBwI`L_12S3Sqv& z-`)X69AS!wllx5Fc_6(^QmSFS02z7*j}iZvScfE#8c6b>M`4fJFfM68|FQ#15IFQ# zHqI4-gfGb}moHzQb4(eF3KUE>>iGCT_)F#gVDHW2YW}|eQ6-cNk+Dd|5Y0-OjxoZK zp{NX@G%2YRksL$D5{1%8lc)?$N)yqfIZ881Nh8gp``oW{PQ&N(xWC7}-_PfF?|t0I z`yWoP^E!L4z4qE`UTfQCHg*(->IA?q9re*HvETX(EJ)MTyEc!7GV^q=jolEk{n9Lf z?`ir4<0Z6tQkrGq^OXN~nZGZ=Q4#LF5AXB}jg=53zYjlANy?+OLZPdF_C!hT8$g z+B{oQkD7Lmui^S}X75d5Xq9a-3*CAiYd>#F%O9l zFAvoCS3VhMX*GB9>aU|bZk8IWMcoW9%UdqnGiUL`vD>rP)Q7we6L~J+3g+q)YV0U7 zH|Xagjh0Q#{a-Il7BDdBqOU)FJ@C=X_WGi&0c!WAUX@C45?G~X7WaEoDYXA3zl-bg z18&B_b6nFa;9PJ+iJquRVihr9a#{8Kf(QplJ9R=W_)@#m4bhc5X34HP_?Wd320 z%%g5^{%QjWdH(t=VM6ADKebqB`7{6 zT`+jLK6qi=J`I=Z1Q0Vx$$TksP4i>kQ~8+V0%bi=<+F5t4Uk;grmH9YAOlitXh#|l zh4tKt`-=%4lMoaB;`~GJ;y`H{!k;|R6FHugWT5sYn|N6bR>s1O& zkW~+?uO1g{l_YxjMDIQvUOqIiaiHyV(`+2z+NgkS8{ktcGfv$## zyI%EF+JGqcj3;2H*L%iGCraQ0PwOv9Yf9%wULn%)Jk`bLT_PshACi7|qOFXSr_zL( zKCuJuLgOaXz;m}5%wjPTJ*1j#5uDF}H|j`}Lh|3{xqt((f~s`F`{;fT!V$s6iB zI||Cp5OmR;l5f1Ot^_A@am-VC)aR=qo$Peb!u5a3EOXL{t6AZG96z{kcJz@CH=+z0 zV(Y`C3nhx*<@EE4$EjDGYu(;DLp%K_5`1dT&yxG}8_1Bo8pfCrTIT;KzPfg~M~fDu z!-mlY*8|_WBQz&y29C93j_kp$J!M)u3N6nPrF2oO5MGk#HWhCPVPTdGlvwQAJ$ryLS^|9 zhD+E&%wqj@gc9lG;9qw4pu9%aq&CKTW|GBO@k{b0rhg{iY(KQXSpTNLD&u`k56`r+ zO!Hzo$>7mL^2W=#ygkVN`4|ArEo%AFHOE4Q{qLGOPg{^TrhzotV8_0gp_R4ka?a}5D|NStbN3H zO^XkH@M5Y#fy}g&HvPGpN)epaAM(qf(6s&ehH-0oYq-9h+3Ue%$6E(_U2Z`5sY@K< zso}CbFl##53pVzt<3G4FGmnwIIH;7uEQ1`y`G}-&|N3#Xh3rKI*$YQ+K`M|^O~NXZ zUT%GT4~jaDQTCGzVtTh`_blv2l7oBiV$)3IOle0QH&WQ%tArGXW!O^CSBn8gTzxeyI&~?cx_lubJzZJy9;Wyy1UX@ z>%gt~2|+nm25##28!Wj#@|RTl+gyn0gpiY7hU>D&2f?F6U*3cZ zV?V;G`rb3I%==-|zSJufx>)9!1IVR*oD;5Dpe7UQ>&FlXeNq(|4_Bzghv|F2h~+I` z5AV_)2o3jC54jU!J@MZoJ|mqARHN@5b}tDIv!h7eaFaEC_|qmdQVD^tP3AeaE6O{L z-c~x6nckD2e8?<*YSB0xh?ndG;rX6lz-jN0r9ZOX{ynk#%QQW^9Y_SjP4nhUwoBX} zh~Ho=?@ns{ok^z$c|TU$tut^WxTyB|>mZ*uuL%hs$Dd90dpkMwVO8LF$FZ{`)@^*O zgtQQgtW`yetnKVSj~830>Lk-_iM+Av4*IW*h(W*Kx>!GIYb98MU#xSZuYN5Kfh$rC z+*L{bj#kuhcf$+wORawe2W7m!&Y$%$()OC>lx4w-_rA`13E#^Wpn7+Hz=w*WTH1Q? zLTroE*duM@dai3GoI8Vb^+W!2uQk`#haHMsb4T-PUZ4+0Xp_{ru9k>?!!KJ#ixx&f z$1{REwq5;a6e%yrfZ1v6)C)3@Sw2t8=L7iM$Le&KEK5&ZVRYHSFl23#=<~SzCy&FC zkR9$}@;qMC+xFKoOO*l#hdPA*7ZujGw?3Qid%V=|w(|1mGK-sye_%mGnUya`AS$Z8 z&WgBmoK~^5?A3M`FP#*6>!}+u+K+N(^38Wk-VHuPNWP6=9!(l{T9FS1E8SUnr`(rG z)0MOtx4w}*;vllq`HJo={A3g-vwK02MlFJ1yELx8hD6^e8nxOK$-05J=BZRD_CN7+ zdZm#hQEC42_xGFP8R!c;_1JLYKJAdGXTA z`IzeHEi(_Ee$-eF#<1%QT)^s0F||Ua`@EWtSHx+_bVVS^vh^`j}3N}e2e z6NO-O#kprlSA6_=*DF))mo+i9-u$7v{1{KqMLb%6Rjo3?PP#%nQLs&6fyezhbGCj? zC0W&Q@@;R|e0fwLEsyV!rvH|Rt>W=}4%)SUejip}gcF7Fb>*T~wZgt!QNxW;2qA z2=1FINh3nm85R&{%!&!)iO5YliV=h^-G8j0a#4PbXZ88WKi4$>nrnYsD|)jGQa8%o z=&S0bRmBnT_7`)D>s~1wu!Lt7(_SwIdhjJp{)<6!7EIU5=U)&eXUYOg%9BxqwW5(#|c7 zPoIU|#WuXPE`eKRXMxvvMFy_*;o-riUP7%@@SRmk1bU1WM(? zAyAD3C5hhri4%j6M*SY5Ni})l(Nw2ZlT8BVhPqa_Ssp#uoCPuV)|1>P^;^`=J@7~ zpOCI1n{_NgyK&RAihZT)Co3(osW=nVu3pjzedOlYg&}KsCepWgD{2YI9(U`A-E+e5 z#oE#H3)L3ET7MZg&{H2wmvd4Y=&H(pwpYBgx2LM>rdVv6b!K{YaFF+Mei6FNj+J}E zJg2-4)`<=~v*L3KG_YME>zo|)ykQsZ2?|mQtZK^hyV_6@U#eL9<>>8uBrP14=x?09 zn*+LKCD0}|Bu^vxx$WjDrs%uY6#MLp{92@y`pU>yc9z?erUWqp5lP5^FrOQGqaeo8 zptMyp-u4t^j6g*CCxj#fNsQ_EJx7uUdu&- zkaTI2KbnHOqi1hh}u%R5Q9^-CzDEwKbihZhn_?=iRNJoWW0JLn3;RH!nuP2rflu1lCIFO33iu zC>n2gi2l%FN1JMIVTQDAIZA6NFLmh$sB@0XXZHLp6?y0&id2<_a}}(&%~}3C5~T=I z2<4TYF;+%?bLq=H3IgG*bap1`Arp*qne^dgC5I3G_a+&435 zu87~X*%)2KqIO%Yl!-HcKSlM6Q&7nn6b0OhjIdpj&8vnW?zCT~6V2;pp}6h*YzbZ3 zk=L^it#Y-5-$HP@c8G|v#siW(X}h#BD(t4RL1AIQ)|tt%>#mG1Og6$@-%avT7z9_S zo!zDq-cuYYYgduj^GlSJd9<6CnA$nX7(P1uZco;B2?GR-cfoh8^O~M~5*frXJH1wD z{9Qbk_g$-ytFDtaGHO&Mg8YMI_cilp@)X)ikdA-C*YXYvikxl69MaQ<<5jXw*X+zZwUrO{TU<9?gd~m! z%62_poB~RV!wqgn952p)B6CDZb+i;LZ|SGc{hxWaLIT{>jL#cCzh|_SNj*wM;$S=$ zu3r`%8Jj~=tk3Rf4rO>F$wtGtFvRF=POM+&yw&Euef`b$(Lysq@B6)d>_T6YZmn3J z4Rc1AL~1nD^DN=cG+!uTku6{AA7=%H+~Ib1IBY=c?0J8?R4HMIB8cF1yKJ|kmE6Wl z>jchio-UBLe3|yCWA85W7@e2`WKC*8Y3P)Rd4{vnOCsJmYtGs-Gb2B#*sinO;Ie_t zhQp=;0`92sB%vm|A>D_>vvvDwYzz3Ho9#{?RUNkUcFl!e7Jbu7^j?!9S4! zHJjWqzmSRjJ?{{o4);HXRDAu zvNV)!&~8pKzFXyiE`6Zr!W7DM>eE4Ql^vYqC=Y8Mlt&a7uSmTm)MJLChP<0j+TQm{j13-bT7j=Ia%b?c~;b!$a9LU4Zko~kW#3g(#_(&(4;pbu=H zlil?&r&ek6+c_IEI%}PD=_#tK%uh_s-(ughun)z{f`^0PReu||$Ph)eaELOsp;R{nC*_vVJq<4# zzKS{D*Y$hThv4d`E6sZHYw5L*at_3f7dPt>&5G&itLxFxK?q9^A;bV74~Hxg(Kjhd z=|D8vQ+AtXL-n+A5o-!jiR=D-Ztc~Gb#9ux`QsG}YY-7CS~5;9q+cs~mH&fa?c0b% zW*4;zMS!(dw^}}kOBgrD`~p&QH(lMx>kGV$5*tr7BtAks^Mm~y7m!d#{>P8i>xhO_ z&70Ojr7Z&galrq(H`cevZlncafOEerPf#3|*-6hTz4JFINHB?bcr()YN|VW{+h}Th zcU+f`v}VXY%3XU7`3%cMzv=yY0#lcGE@x?|OL;x0i-%UZw<)35%uh-!u=Q?bf$jz* zQv4tZLhvMuBsLn`=lgFVbu80)2%X(CrTSvs+b2vvlxurj}+iMUVjGk*59*)T1 zY*Y>3nj?>U`C?);*Nd#FL=?Z|$@pq`+ShCMD3U6j*(&yCm*m$xKd0Edc0$dQjR}PD z?NL#5)TN&;ym5yZW;;<7z~4Uo*m;V-4c0slRzb7bo;=vz9i3fENZN5VVS9A97|PP4 zc@t0Q8IBdv%?Ls(LXWJOGI1Ie=Si3;D3I2L+MER>OcwhLA>DJRdeR!BF$1+|m&Q%L zZ$HrLbIiUj<#fZ#)C?&5^N~vXfMi{j8#&!3MZ$0aow$4{!Z;I^tGf=qfuXn+sVzh= z%YS}I1@%w*8Lh9BR-gH!xA3Q*icrzl-LZixwnZ5mkiC2#rmnxj%4b!`l5l}(xrt=$ z%7*wq@`u^0E%v5C%FlS$<0ChlOnyAxP>}09cAlCsg6&f$w*QQegld?#k2mp>h`>I` zv2l6`NWFqbFCzIk^4HoLig%kObRXNR;ht}8c%}-%TF(3IL<6JM%MTP7Z`MAbmMn}Y z@5$-sw-yEMO1;c8xYEj)3IIKxW>p zQP(?SP!hjzwd-lwh)NV8A1jEQlAHdyHEjCpkhd!)N*T=(E44=1@ePNUQyU+egh@V) zTHH~dP}5^6XZ2GZ!HehenXHJmaZ~t|71-{#+LJetUSRKuPtkL)PYmVN+CX~j?b&<( z%tq1*ahm2Gc}n=f6EM7_#F>;ioH@ifmY6ZZxFro98JbhqN;qtn@y--;?QX3Vd@fL! zYkxRcV;!86dY2Ekea?x3x3TJK2TZMggHf=vToDTstZHdZxSp`eHT?ANBc#aZu9ARK=Wte`SvC&qv-|JJkeKJu8cJoz6G^ zTw}O;$NCe5cNt35`@7JmhHARp4$P?qW7v4<_Cc=SFbZmuB|j^5mj&$lHjCY6i{T5k zk0(P-ViI9d99(hyxi`vC{@5P2-WJw)4m>5NH*Ha6ovjNO6o8(lxYbmfgs+ugb*!@L z!2l+U4WgkZ@vhgzN{JGobc-;>*T%P~0P?V=kKM%OwZpE7UHJ2oxG3v?82rJ6v_kyo zRbbURih1%OoXnGVM3ULkkOG!;iWD5DK*7fJot-md4m8wpZFoOm>cwKB=5BH{p;pVy zuBsmCyra}AlGT%SN?(&eG7F63^2yJ$q?sz0JO0$`_;!H0Ltt1daIPanDinUqsV3@m z2zd*A4XA17lQ`Mq4@aI*e^LLCY<1a5%zndvmg0xEyUJKy;0N0~o<>|ab@YZ<%9lm{ zc?BiU#ZjP$qU6D*o+JCek@p@GMgL)aMhH>uB}gp4>Z#;0<{l5W=BY+P6J5DIhP31( z)c>AX^&t;%mhqPsu?`+O@lqSQ4qZ}3Hr)K6&fZ=QuHjY8tl()3tMSKRf6x{*0dyL7 z&5HD)pk^4+uX(Uf-N@)Ifn)=LOs43b771D%CR>rBaWs=P257{k_^Qp65z-aX_8JM) z>=RdUAxh0F^46-kGl}HjzPuU3U7K<95%m`Z$rn)em(|Rc?`fn8D3 z>3SPJxp@I@qJ?MdQ*8y7mGl>et9Qeij|v?27}kb-o2Dhb}a8?iP5qXMq5fg z@?COIAV=TAeaq+S=W1eiV>0x@EZi2{3%=+vNBFJS_-m4!N6sl9Nf)la5YTkT)i6BT zzbU?`#3Z!2Mfs_P(Y!+K%3kF}yL+D|<0dKLhzm#PKf6}$xG;)Q!FfYIzOIbA7HK-q z(waTf?(ZDDId%ruG|L&Q<8Vj!D8?iOiy0>8hhbL_9-5Fa^*YC3{#i!toEV4oFEQhA%A`}dCs?%CMmKm z!+H^GRT0a8rDI`HxZ#%Oh__Y+9z!F1y!av!7#dJcSnRwTzX(bQXWfIEiojr@N)}8w{v94R>=G(`oo~iY<#eu9+3LB{R-bRMJ$Tw>X@VaQB2D{LWz}BwfWPex3=7Av|3DYuS=+4H%yl>(<)+{OXy%*Ao%Q;06z!vMY*WoOsoD zJ6OA{W#n`WWeh13EV)XW$=drNDmprUuVi@Q7xL|1Ou@>1d4_r>&)mcxV9B{UfKtv& zkZ@<;1sLbXJE`Rq#xPUTMQ~Wot8#>&JMRP=Uzp&>6r!6k-om%GD;OL=#aLP!H(4=} zh4V0kvT+k7^dL#8k*-00^Uv2;AaaQ;w*)X6N$=Hxmf*Vy?+fgfmZo0(atN&5eqQC& zG)NVf-Ic|xK@IUz?r$A${5T#fvMq?A#Qbt72*FEN__4?3UI78N!u&KPux8^$sqYHR zTYYaqx=2qANu4^2h8ad}xXprO#D)RAg5>!>=>2!}nrNPy!|nw@QT;!t{{I;zI|(U0 zl$Dd~OiYXB;?@HnOMlmd8w;4HS`F&o81gw+73|{n!?WbEuPlyb$y90aL3VKp59+55 z5Pg+V1`ZL!Z>~&zIqM*f-Ra#s7D>2*qp08Yrj95E{0SAeU`pl>GPXGwX6j@H!RrDX zo`pIg2Y(qq!`=DXTx($TDH!(+0D$ z>bs8%>uDzdRE3opQCJx_z^#^#|M-M8E(a$spyZuXL9s5vif(l|F}3Z5FjH=E)xYE# z3q5r9sJw+tNH{p{FhtLjn5zWLX_kFs$#Y@wN&02iM&`Y*+*j;}0E0vB@tsWM zZiA4L4u}e;C?iP*lvJ=<;mQaJh>?aFufo3A^rYNSkS^l_=Q!`v!lo5=hEd(fQg96$ zQ1R$vn~E?vUnMCt$GITyGOPHSLgH&@vHn)U@XmwG{n{c)S41z}3 zKZMwaiDoX)3eT+Adl24y>bmtuI|D;|n>fpTQWoub^ROt zx8B~mLfJ#CzuK$*L_bq^STbEb)7`c7Fq538g4njW&0XjF04o!hm4ig52&@ptQmZnT zAEWXSz2V3&YaA<@A|v$nob5W3E0uKmNQ+CM=x^S_7VCa`OZO{V~@4?a~ts%_i`v)GN(7FOXdhb%(~48(Yrmok z#+Iv-=N;_I6dXEVFC=(7ho}V&2qfB9@Nu9zR}r$E>%yUvB=nzN&dU z5UA?i?$gU$Ie|XOm9sY6Yb(ks-fdSs-_l=H&{TX=&aP=R1*Aj(hIK8;hIa!SS^%lE zt*T+#2pK>oA96(CT;GuASRXBg$uIIAS338@zQ1u{soXNfmZZ^j-Y7V#vz*7q8aJQY zy00jC+_Lv?bD!Km_gB>u0p&J1mM z_Yku0j_l0;F>^;{Tg+sNCD0~e^f|w>nf{(CQ2VX=`Z}&cCm5k~K-k1fOqu#~}m4sI9%}6Qr2dk~RU)1f)JbBm%+FnL$ zZO$RlZ=MP^Vh7?*OCGSZf2N|KVy|*wO?yXZ!k^g~o0~6f0W5B5l z(#FC~e#f}8hRcN^5tQhqK=_>&=+UCb8mho3hU4V9^k}8?^VsGJUf;;G|9tnh*uD(+ z$H$~5P;;;i0S?YaXaNxqf*nh+TqBXK9;}_aM|_ww+)iS6^=mi@^ViP0L}q@)bG+=7 zgsE4TdP&@$XcZQ6@P)>uPejeAg#U1w?3nUUyPMLB8t zmcsN6YP+1NN*%mdP>7YB||UO6#UKdh9iTU#co%$R%0li4C~XDnZdkMbhHrE~M0U&_QRWFV1?@&IK2||& z>O1*SL9{zi4hsK-+-D9`j3I`@*BJWGQ2CMq&FT`XPuKH7hBKR-9uCfA zj5^`Ouj^b6#5J#$Z%HixU(6-k_+pGF!D1K=N7wwp)nm0z7}3{#1p4^M9RDhB@NBp@g`z6%2zbH^tP`S$ykNKinUa># zr{?1SQ`LHE%-!`}i_IOoK5{DHTmr@3eN-{q+l?mz*i@s{`=7BTD)=)I^T{oD$-1~@ z7MP>GXhcj*ahhzQa4w-KoiimYntC#%sxvME?(yT3mof>8{B}8ZAy`6YA&Xq_#n1Tw zc68QC>K$Kmfc0*))WKnn4tz1p3-V+y=(?g|8=bjSf)Wz&n7T`1Gy#pmHsd$hTRCe; zM?7(Z31nb=vUbWooIQh6qI=n1lCjX>X1f+7<41f@9YwZ%nlZF95;w3{OGAVPEe@LR zYRvO}fg~R_>ORnef$6%*!V(YhqG5y((+uP08+6?enHmZ}%o@M%W~9{%E+9{B%RAiIL>5IOvzy=1aRU}F;S@}G_jCTx5bq$4 zE<09yAAB-Q<>KQ8hWQs9+o}oj+HGW>!nEtSevxTa)(c{E;j7rfNv90hn1SJ@`n6R2 zuY?k^;fvSpQ0_b3^X|9H2t`de*8x8glrLf00GT)unI@;V`>p_-ul`6-mKhKWY7kNX zq6x$%5shH<5!-1L{)Tt}i#hNa?!nIH>4w5q9JB+w-vJxrC#RieA>j>%Xgp>j4Se1r zMn_;6+i)CQIqL&M7U(NU%bP*{txL@AgUKeG`Z$xW_pTbT2!)3681JHppKmNEb2eXr zAM=}kgOTQkR~^TitR!2>4mHgb14d8NQkaSvR|yEpNKVHC$3}M>&>(qAi4~jrx79`v zs^8=k#r&cbW48(%ZCf6agT+$eMh?M?!VSnxevp z6m|04wuQM88RDeTeWD3Lk;DaL@Cz$2ssG={+(Ss?%!L%Dyod&ijTv*}Hup4WZM!$~ zrLYL%JIL|<$FdZ%=yg~$_r{!mBNf)zra)JO*YifQXkil5%`7uLK+f%4d3PpT<}-xF z5%g9)bOTnygM}}lS#u3;EyH1qSaVZgdAV1ygP@>?k;&1K7Cu)AU^H5&szc4kI0Fhe zGx0f(D;)8=HH89Ml8_SsbB!;3N&v=e2DELrhomN-`d1xsr3VSEzgokyeAb+oviqI9 zBa>~sQM*MG1lmli4VkV52_om7e_O?P1!yxel4Y2iLL;^#?Xb>0?Dc3+>V#``JDJ}Q z`r*VAt_kHCgHPsT7^g65VV>on`*IY+7EgYqXP|rz)Sye? z&Yt(w+uxN1~(_2EG`I_SAQLT}43H0y5Zm+w)39Sd! z1uqoY%@REmvAHs{p5DXz5)5F5E6s_G8MyFXU{jO7X*)7E6<;KI;+;7T4=6Z&r(pa| z7b7;gVCL;9#>*&7(|n0CEwD?91c@@RmCMBW`G*vnCkUbk1x92J+d9PS>GldX6hw?& z*fq87Df_v0?aKTu%7#S`q>#?lybm*sJm>@vXJTksj&foieey)&4#3=y>%=@n{8;0* zYmG}sD2wE+A(t>?AC~a)PZUCI)Y3q<#53wJqg|2xfP+&XgN>>!-RquzV$>a}+K2f^#-psv#tl3Fv*5(o zUN&~gzocf|05r>_W~|4i`<}*}-xo>=vj-l&#nAR)Ou)=nEAto z!+e}_KtS1s?@WIP5YF~<0?1H&6>CoAqTRQy^#8Vn zHi#bTgAB1^3||xye#~Gcmz?_s=&y{^V^NLo0W9A-Jd$`2Qi#5@yNH#IGKdfMhyMbj z9>NAW{5vuArc05OtCk&h<%MRkOi{RsbsQT(B@rH>S@@v@BwV&ztoIQ2VCxc0b@4wJ zR&2GCd88dNwTwVWB!QWouyqKAB>fx5?AuH{8xqT-{J7~RpjQ$^lk%w@X?Y_S$%gSC z)%fFoPc<$rEj2PRF&S9+oQH4e2JC^e@GYib=6;Wpdi_C+BTc(cA?bGO0m zDIq4U!l6=1K_<$%TTE#zap|qGkXp@>U*1$vMRG(hY_p3f`Y^Oi9qW^+bGb{qa zJOb-L`j>B&=Tp3ykbD(~&U0g6=4q8QNlpU_)x+^PtIpjEnCS*@{!^?l=oPv^Bca2U z{)Xd*4#P*nn+0Q-y74EZ`0ds?QsYRRP0LK*uq6F3d7=()3w_nj+3YkDm@s!R=EjsG z!wiZ`(0?b;e=(>NC;SG#=-{3}8zJ5B)hrxRgZ+ngK=#s}VT{lvCgMYI4f?IALC9&1 z0i+fpU(ZqRNW_7LS^~W{>n_FeA1zX%Vw+fhE(x_Rbo)r;*;m;6PV<NT61qU8=4m&M~!_YRKA#k~TCx0YHpl_<+3hwa#W znJIpYshN}5i9PM)3Pw3c$*t7|AhP*Xo zKQjo-qLT{%v+oSds(n{VWN1t&kV!`7hdo2cZ~%{;t^KK$S~=}Je$hAE?LyK)g2D7I z$|s7iBwpc^r6mbre;J!FhQP!^?V&N2OzrCUsIo05!P?lm#AULIcY=9e^?#ewvgI_B z9{>0HPFCBt8J3ildH&;__+NP^2FZ&0Fp?>lfyY_$PMbI7|3&FPM4+Hx^%!ud9~=VA zJohp3FgAiJbK{KpRa5o7w2Xlcf1 z8D_@O>nk{JE7i|!@a4nfbr&VZGI7mE!ZQbJWZ`5>;=>7DztooX4Vz0d*%?-DNKzvk z&zVv|E28=ysWGcx0}l2pE!~xg8QLWbG10+zF}Tq-Ql-H0YA{g<0fQChXKdseo0>9Z z%f5WsrXx6*!NG*(U-e~HfV=y4+!EUV9k3a%IV)}JV%og+`uK318|X@Jo2SzE{=dyV zgo&QW%wA!kau-mk*KnWsD;X(-rA_`~s{w(tXRgmTO16$-Jj7#TdQJn-l?K9a$*D*m zqA6|SdYivdfh8R9#Xh(X!^_g9Q}3|z%T6r94sJFoWeMA*sH+HivU&owon?-_c=DQ{{C37(VdZns6it{W)biR7aJ1E^m< zDd#63N`t6(nNpcI1e^e&1>p^%!)SrHV8$$ZKs{k(V5gV!X1;!e|JQvm)nbW%LK=7T zC3r!Rzo4ZAi-@`mMRS%C{q_BxOgM&_6id@3j*xHLm6*C{H3WCM*M(M|25e?{wBuk> zj2~#@R!03dfM=+QF^UaW!u(~AF%@Hwq^H3rgK9bTWGt-v*`A+9^GtvRAU!K!9Gi?{ zvCmem7OTiOeb1myANog~X5#dJi#nYYeQ4mYZAaw1tiaQMzPZ%#rtZl8Z#(7LN0_4H z|Ihh)axWy=9QvDHt|8p2(buC5v-_G== z>j-)ro%f@Ah>>uIz(x#T;#e8z0cp?yrRD+L=rufZQIbb<0PBi5S~&GMjCSBCl8a%WWay& zr%Fy{9Kz$il6P#kc9K74a!kgd{?ed%h! z->$L~S@zWwM`AJFF&J~&9H%_$hcx0u_ck|M_^&~UGRWXs&E?8es)atL{doD@kl`vE zm{d8?&|CNB`QOtKKU8{iQwtvKIAMgRROkyjU$Z)Q1pRG|KM~gN886hL;M1)Z+nm;M@K7(?@WPfe z-eGgb^S)nA4XQf|t0F1VNyM8^s#w^Z-5qo%HNTQjZ&E0JVixe7#N633DQtXSU1#E* z3i-3g%x@i4bqcKP&pa_{U_VQ>U=FT`!G9lDVmgYdNx)=r^M1||t|1tMLunQalBD$T z_Lqx8^Vsq#w*Crkn=Ib%+u47%yeoj;N6qK#PyVxZjh4f}8BZ*EiD?TdbTC$7m_jZq z1;!MDu=nlklbUoMi|noIav~V)06fK8Q)-9i;_mzYy}K>ZyT9ivUt(wViJ&)TDl%P` zN<)cxqZlNboD$xyN6qbwV73C*`8-%LI~Y1*O+_45 zaSlb~uNP18`RT5(;i|~jnCSEO6Mh#MjMEu-c3a7vE~Fs&rB(47!2okJ86zT!Zctoav3GLY4b*Ad|^13!HRqz;tClyz4K zo_K9qPwVmVv0r0qdc3UbUMFiD^7b?m7$3};P{P2rzA;ylHUMe-YZX{YM8f7M%*}RO zK=IF{<@oN|(irx4V8X`-`I5Hax8I_FcV!oa_1%;oU=t`7r|ts`+PS+kgGhs1b)@@} zn<$}u<8$ z=uD;hD9-Gyl?~loJ^0WTZOL|*=&rfir#fUt#!OZj|ZmeOUuk9k%%5|q>a1Gb}l$%iwOu207cXm2E5k6;m*xX3> z`SZ@czO5q4Udo-hshxclGY4q~PEsT_>^{F~33H(buuri!V!V^AFj;|?0tqgVmxmXy zgun<0U*P(27zQ^d_I>Vd6UjLjAO5~srMt8)t?}xb%G$9*#L1OGGF?P?T_|E?5Ocuc zZrhxp)I=8chNq8~!7{fTeJy$zv<;#ywJLq!NA7Z+@y7-}@|RBb4^rt4plollYU*9z zbwJJd_0lAp8{3N{mT<1LqLJn3!h9q}(t03@_}&lU{RE4qe_=)kw^er4NsEgA{haEp zCUVDL@t&cqNqTT%jvOWCgzyxA##Ur2h;4uL~t;YrH)N$=OL zp|sUe3=b}`#wLH2p47x`53I#1+sYm1l|`Kr?tfeBL-!CAlso@p_^u91Va9OdF`TGy z%w^gVEMQxU6FVVru)D(WvQ7$2KAo>8&j0bU`qlX0{sK=$>jif0FGU_4tR3bN=E@*( zPL+lHlo()yVU7bI_V3VLgE{I#yx3OR@Irr0Fe9xg8T!L^DwA-dEW(Y-8~P85dk<9G z(?z11OY2R1h6@g)8qhZ$p3U?T3{sXJtjhEYO`Dn9c_9fY9di6|%{`CHT^G+)h>Nf7 z989($JV*?i5c@}pnQ5_aBS>rd;v&8@^t^CMEe~enpIyQG6X{R|H?y|0p98f0Q@ScE zdym=n_7})$mD$u@&2K)}lUT;C8(?B=WZLSW_VEW|qB}N+I zQVlXd~#O+Th_oquAO5F0b)2FA=syb@@U^NJnPJY2V z6fCFU!ju&8U_c#rf6Fua0CKmq1PdTOWOa%dwvg2%J;XOAoUtT|$Xc*DBeCmSWzd7{ zILY0sPqw{nYfJwtbGDh8cSuU=Hqm%Q zhfe5!?o1VH`FsA)ww}CF@j#bUy8mUZMHKEY?Kw=rYgSKzOK*k?CiEV9|-15 z(7zn^SMX{@KfonC*aJl(A?e=V$N@))04sCRfYRu&_==XquPO;|Nc|d3*fXh)*WhoY zn`NbHLC4#L-#Vh1X3Q)dT^qP}2Y&5Rn<*aD_Uz-|o}LT?qKxUPfwGoGdNI>h1Hm5L zQ;L_3=vzbHV%s5rj05tL)4@z1ZGdE*%DiQpl`I+)O+I*zQqW7#Z_1sl%tX6#)7@@O<4Rayw7NiWjE>oWoBAxP=$_ zWMyUH4$DVYxcbBY+4EsVD-@Wkcp4fS{LzEWAH_#O&z^Vogtt~)v8s!a>*y&VX3{>e zSA&(xa?I1d0JGh=^+Ek#iX5a!R*lJYmK+mX1Yt)p=GF?>5H>QB6cl1#LJls^#xG|VlOr^ z>G%icbA53AI`@9e79;lU1Z&HZ-hi~H65nmeT@N(Ru! zaR-=f^N1Qaax%Ch7PX*fToBqnb_U+F&i35#;CxpN>$MS~e)aN6q803303;*VJ;v?E zFv&_ZnjtlUMsiB;jWayQ$ruwQ6fa_0ZeMgUsjyJfw)&20KCW|_soXha;V>atZr;&Q z+=k|kL0(>7SEh^iH&~UN>5spXLz+YlX8cj_x{^AWV`6~|%y9rxo2f$%f^$ZCqX>W8 zQ5=WCs1?MF<5AUo)7x`(z6>NWM%VpLoi|u($f8*|vNwMH6ui5RTo|;k^U}OTFYC%P z_RM<8T#^a(+_RTD9kdv%>&LO9QuvSA83{u$w&5@LuXEAzgOLGJSw114C5!K+r>LI}P z%e+~)F4(gxxyf4D207MBx{8!{yL;0%FdY4^7MO}O>+I`C)!~-Os6TvXFs#;##ci3$$V^OTYGL>=({~51wDTOIb$7FfaZSoX?=| zEw|{xPm&n7=WX@q6mmDUuTHzumUsKS)rgvEvR*!|uej)ep4jz4{1_JsoqfdmyBe8! zg8x%2^BkWe?`GDUd5Kp?=juCo!Jf2h+#^YLI-g^6%coci~&YY>- zN(hi`u`tPsWz3MYWPZ7`9xOTLjy;tc_w_W6%N}-B3Ev*ws`Z4d=6RSV#>BYOy%pVM zWo3`~oEfImkwF*mHZ(N_*812hH#Rh6&QgB;N4%!tVD|${Quh5nr)mjs)MolPWm~JL z_RsdqYJVlVv5M}PiUdJ-iwP?Dj%0tfm#^*d*(&Z$S9l(8(;rGY(Tq?ZX?fV_jj2ap zf5rcA?MD@|#O5FHo9b{IVg?x#2`aA?&VNnhMSI@lfY5RFN29uwI?G&eEg@rMswY#- z9JKFmZ^Nyv*-OO|f8Vs~zH%th$L<@^b=H7IStVI4!$+_{lH+#Q_txxTQAK>A1`U4M z{4vr*XH|yu>XDL`WTbJ9Q$D%Ew8BzvlQ|mrTA^wFV6pVzUUU0f;nJ11U(cRrFSR~D za6T5c3X8=4Ply}2$F0#UBWYN>8}*ksRUf|1_M*`kqjxAe%Qh>^l=iXenFp)W0U(R} zY_4l44zD9kf_FB=*~~v++vND4&WnW)FG-{(S_388YKbM%_eFunkPFpQDJh^1CnjB= zFm?zyDGz!<0Fsblt6uHmV5Y@`fJCfi3ExGi2Xlj*m73Kiw) zjgK#guZ>kLPxg8amTd7#f`^bC%&6dHE5oNk^s~CUQ9P5v!hK+2g`5S*sT|YCdc-i3 z4P1%4&30YQ%@sJn@^)`n2UrX5y!xX+T*IGXE$k8WddqVeHYd;QHvuE~u`kgMRV5zq z3%?+pHPp6=F_XVEa1!=Re{lE`{^#N?i)8{rlLr=l=p80I@QEOLxAw8c=rqJ?w=qPL zLq1b>4J<|b?DE)|!)lfB#VdfQ_rr#`%$^dXLw3z2er}QhCuO~rwIJ5~s=$_$u9yf- zcQHQQbKB1}uAdUjEf-5Bsogv?UY}y^nFUatk^XKGIdy||0E1TR5Frmm&}9zaabXeM!wJz$%=O|8a?@QpmoOhho8fUUW2_tP&V zso|5TWDL_YyNA@=XzX{G2rn%E3iEoF_xJ~>+tuaD7Bcft+8x_69XxfzifVsi^9#x| z=TgY=wE{-+`K1o&;s3O{QkEH(2LA1Guabm#pO6r5$Z&rg)EoJF!z)%(Sjc4OQE25n2tbgIFx@R(_~vL+zSp&YNOw{D^SRx z)#2R$5xs;o{C}q)npGt*OubhR1=b_;W_!{|t&Q_b9cG6s_O1}VD1^K z{#%wr<6b0%vbh56{oV8#O9)rkju}@4=xcI- znjlrJ>**Vrdq#R~JpC?pTmt{=GIUsNHoSGN(pWzSc7#; zv|$Dh2WJ3yg{!x7_X7OCfnM3f++vz0CJE`AMErM=0P^!D19MJdR3+iv8Q8%R4)^i$}-4iTxE?`#VM>=o#Isb)de&I;H0ar9}}9<$(4axj)Omkm1H<^pD1;5SLgJ zKVN2D!o%5;3`JgCjS|ySGZb$j+`fH0cg-i!A1Kunn1U%be2ihLfC^k(ma%^HP3}7| z_PN6H5Z2h!$k@Kz4rFYxX&C$6FTcRu?w3{P*+p0#Cs~i>%Z1$vLhnhNan|8DZLzVtMde~~ z%Phq6TMtdRBUEV;K}y+8?2O`{i}A(H4OC->PxFP{c8|Neo11U5BVaOCJ1k6{58-wK zm-x=*W0$(Q+>n}uju$HljdMH|=J{Jh?m0x`&Zn#4Im&OyHZYF! zMu@uc|J7`InJV-R_h1?e{+yyZXG%g_fVf6{C0-M>E-?yfsdm|MU-@sEs`6B`*t$Ro zi|VvpI)@Z?D7EkTg3w4ZF6|m0v?4rH>Km?Q9gnL)4G%rAI`u}EzV|`SS02877cS^# zbDE#qaDi5Ug>Ns2Vdl~KYk~wL10*Fl*GihV{&M5%TJN*AcCt7+=D2^k)3~|V97oQ9hkwZ)85P|2dbq#6=(*v?42CXU zbGeZa$r%IRUH08=6k`zx;I8_oaTHZ;CMPKN*I^m1j4@o68LJxDPO6fnSlp)L(eR3CE$% zY0YeAnX};rkl2)!wNnkB1IV-SQnhLm+kxl56CFSSimNbWP!d&b0O~PPV-C< z*5?czU(_8$0f+y*yc6BgvaF?#Hth@j!XSxD6K9_m`x^+VjgnD->EHJd-y z{Hf_p@r2XY{{BtY_Rz(>dZ7mI5mmGz?8W9&`cfV%4}6N&$UIF+H<~F>QnD#qICmL7 z(9_?rdXqSx#h+y=oTJFeb;U1sABR8EcRfgK?nRr@PvavNxJe&=cbV@{hFGO?Pu^_@ z#8jjw%v}=^C*R&n@d>f9r*R?oeN7*M)#kdZ$!Wqp#1@jB^c}BU-g@%WSI=)q=xtiL zaGyzN_jYvHeQRoe2x;@ zpYhVuV8ewp;ZQs^3cYpml$vmzZuS-tuW3#y16?=jLZV8Hzae%x5jVd+F>H1i{npd$ z$*zvn*A1^eI_y1U7b%ms&}sDo&r^k30Y?==`G4w3)V}&C#(T2;CH*+=_1=2T{%-~D z<;-ua;+=u36ji_IUYe+8YS##;Roq(LbIt|67=Q5h>@S9Xj_>@pxZJpU9)a~KK>e+0 z`7+%8+dzo=V$%>RM0Bceprx-GqJTTRkc#d=yVSci;nG=9J9f`n4y~&n_4gyT$JM1ZUb_ zXjE4E#S@s0!4>K>2?K_r@}k_G0MzNv8z`fM2Rj?!PsP2JWdJlc4sf>V&Z_gIeifrl*!sU_n^_hj7nzTdb$YA*2qY@ckZs9QlyY>m8I z!Msy%=e+eCP*e$9OcPeZKy|@a?(n7E#o+g!8i`Sy|Lb4y{nJyq{r3VP@sW~J6fiX5 zD!e{h{6!rxGB@J$%8Q?oo3xUsggnpaZ}tga*UiTLms6}uO{4h^m5G|YBscu6X^1GG ze+UfgTjo}9Pyb_??-x%~2|V0goY*8&DE z$x~Rc;y&+kZKs?gcvtGlaRL`!1ZL5w=04?<7! z)>wskUzE=(=B?qbk?E{{ePa|O0>{xmbV15`VL?n%{KQX-d0xp7}N(-3}W=`+Hbxz2MjFx%aGZ?&UZiH z-=D0^up*#76WYYAh3Br`3?!+3kR_Qr~ zyz~QO8*V01j&2M$^Ziml{-dv;z~%MNPgMl*q%&$5xbD#n?tQg=7$ zSf$-@DK-r}^5k_sf0B>*hmnjj*~-GRVO%Xnf_t*0@qCA%>l35`K@{+_4}HUGEdo#p01gNk`*`*_VIu#1PcKiF|IEh zS+@F4&c|o)F#0lrLC-j zN~+=a7jj5<1t8ci3lk=%d2I-S=fy*44T-S)R;G7tI zlNKt&FGkLrz9YTK2mc+Jiu9nvobi#!GOeY6^Te%Dkj3rJ3c_bjoU)4aGmv zS06|mMF&b7|9poL!Dh_+2GVl)_dbx~UzyGDP8)f|xK|Q+cr)4Qr-gMH&R2fF^oV~Y zGuLuR&#gbqPjwI*%T|h89S>M2cczR`YcqLwu*m_aNhPwK$e(9;uZ~<7GM$q(3hgQ=2=-b;&F5Wv4rPKEFy-x*X2AUHO z9MWCdu7~TF?-FZQ86Q%|2EUI2e^oqe<|&eE<>lFdgKTaWES3q%-Y5jx=A8`Q$ZUUO z@{@1P;K-y)Vv8*{){Sa__wC1n@wcxzE`HW5-DC1IA%<_rNI5YUtK%?sO)znP*H*ce zZt9A>(%Cc}%AG6H{x^ENCeP$CPTccmHer9;)V0>)La9*o=0jPTqF0xwq|G4h#~`o} z_!#zY+jj0G*b{OEPa1QSWLfFWuJ=WI4f%*_=tqzZ@azh(!W2P3=e@!ME8O~xQ7eFy ziF}}v!Q^V(p;B?f(kkyH*>rMSRCt5(D5`M|JEhi zc7qnqDUV9H*gH(x|2#6^UD3l>GJWsf5{Tt#}|5<(Fst@|P~^l^jgs zt9>68|3rs73QV}4cOVWZU*?vdF9?)4=EH%srznSxXDK~BcH{;m8}VV+o(&x`pw-+L z$itRawzjm(=`vr014;vGMca7WL2E>onWUU=AS;v-qsbRRtND6yWl)Z-$5N^G1{^tW zXRQ8Gywo)J>O&dPsGFa;YV~yNm7*%fz2=L8axHfV{o1SPHjHih2z_2-3>S+Nlp^U( zT$3{fpRcs*EZh6t%V$#i7HL#hrja;&8b8g4uvdzFBu3X*#aCy(@BF9Y%tiN@Hu$;F zW7@5+AY(Ve+Av8z@W!Dtg#&=#mGXQf>f-HpWe{_$T{nKX1=9zg9%VJq5;C`7m{lEa zV-{#GPc|?k`gI+OFh^hLMs%Pe=2MiI>M1JSl}`XOcue0Q)aqnhr;&0__;&_FkI9p_ zjo>(A+nzd4!$$)~$}F=dck=CpY)$pqJ2;c~Bqd)JL)KM!E-!k`WBEWqtPt>C{4xQd zj;|VG-~WLi?Tms7_OG>J(m@`OHn@ zjOBq+tyQ2R(O=?972P{nh;SOGW___ryK3;eY73Y2$Sq*R@eaXK_A!%Dh@BWYytL1U zwF~_Ht!QSeQaExsV3%l1oY8*GVE2KiW)w|t#_YWs@8}A(IK#onbHf?M-4Gv8Ksw@- zWC1y?ADwYe@SCd9t8B|!2U69f#}|L z!^*a7buM!>w@o9vx-my`ALp+b*r?@@7z|vEUEx0G zCGBUE^Ch(mbyvnIn60xf8?uGBGVJ}V?bN8g^L@+0eg>Q{P-n}#Iu`U9G;M;rQe^TL zT;$S!HYBU(xOZZOFsTrgT}yj0aEg6DsO`{epUZ>*@-WE9qlN|s%r*33wt<&Qo zfQA+(Md3HU{L%Chkg?$l*KS+WzvhUT29w3<0Vz%@qc&yY3dC8vP$Z;qYtIe~>Vd~t zq4Ya@43$}BJzU)!EaXaSTmkynYe2y-ves3X5NX2%%Uc>#>&c3Z6w9DF?*{glyDso~ zs4!Mq_y)`o-Dv*~q4wR?izVZp0yC7ykH4ij^E=B-kRyG|4DeqA#7lc}KOK$qUD)Hi za;ZI5&fCkl-zR|0E1yR8u0uxdEakxCSXx|l2gEh9EO7V!TDs*oxUE#;>@&xMII6@lcdb|GCF-uRvE1%Tc9gdo^?{ z;-?0lEl>K*qb%036X8-InyKIHg{7inx{L$%Yq%c$Qe6shith#ff>)-mf341U39Kkr zJ$tfaoBu9m4Kj-3Os+g^r?8&yqr6uQ#HR8NZa)A%TIKz|3j5jg`o^KbER&y3EC7Lp zJ{iZHn}THbXl3KKAYICi$}-#44ikC4?Tw%F7G!8~Yt?3gYx~EnmLXti888oO~;rGGqEIk({(^?9TfDwHQ>ab95 zo5tr}C3qIgw>Gjin=U?6)U}%4JiO0-_BJ5CSmUJlC&NFS8haN^RygDqKTS00+J?zK z>@9=TtOC+SO-tQ+Wvr3%Vu3x~zuzy($*_1#yzj4d8lR`D2^6?q9j4wh%$1Y-Bg-sc zvbQ<_0M3M-hWtHWz#lO749m{muvdryCsy3kfNCDy%v}p-6x!}k^(0G{t<7feCP1XPrZ(vm1idl_wA$(`fGCwy~d$T3``<>#nNz@9+h~r7KlkiLNW(W7o1y+9oFURjzj?GvOTCz$7+x|m}rTllWCOlLC)kCr>yrpHHL5dyopNA;IKs7jVz^% zcb8;VB?9C-Q6RcT8X<>iLJshQdImnbjthl(jc#zUCD{p3vv%ggOL$xfj5i+`Z;O3t z4^ZxA0ZVl;yt$9;8pNy|4o5_udr|OmMmT&|67yqx>~>joLWU%vZ=#W?`hB#^i6R}z`;J+JU)41qXR1*!=b$%P`W zj6ZP{O#D853yyU!hu~FHXcdJP#wlsJ$>u48Mh|7?$dFYYYFJoNijgjGZ0`a9V{V6o z5?aRB~I#1E4|^9`vTvp!~YHKEIbfFTfLBz;hFY434(TS>Pf?u zhdhOQ^L{M=SlvX~T)Cz0XylD6#i#l<`0OSyO<^ zQ{{SMhEW{~%E?yL3SNyRSiT1VeNy&X-_{Ii($18m$FbhOo(K5vQP8wc=Lz=X- zO5zu~;Y=hJ1J`apLCc2MCaN8Bz1#}DAMU4!mv!D&nSAjI9863XFuGai04?g7DbF;n zphyv~Dm#1M1`<}UyU&w)=&k4s53&OI_$J9vPHJCGpfPg2d4|1meIcjHAs79s#^$qq zSF+aq*PUIlu34Z`d75PWYy7DYZ;4cY15Fv&k`h%9*vhlg9S2W*dmoj5@`w9Kn&xCp zm034PPWnNEq9L&1)IuvdrK1o|lxV2(;MyUmECFqCk2F6ZABO||SQTp^^t?WYz}Hr8 z6KT>lKsu%ccHY7(5k!@_0HI}vwV{@*A^SjEY94<(P1c8!1cvLPpSF`GZ*zDcJ@qjY zOiVztp|>YSCJhqgZr}uy!|_h~Ki%$oMa&r~Q|`VPEh)lV`0K@e<+PvfgV8P;dJGjJ zLf5k5__kAq}xt6QOX*o^Yq9QZG={IP%3iP*arIxPek-xI$W?!Zw$Wa=n zUuvm1*?z`bW9LBS?OJ3j&lGVNx0e9ub5b2LBR{V(U>d+-k(wFDiX5NHe6Xe)6y$t0n8k3*#`Bi17?89D~ zdL{M@OM$h_>vT`4*Viks`1VS1SDL9h%h{aRXiA{XUV3T0N`BDsU__|@VAkbcS1&1m zZFUo27Vo!eq>f(D7RkI?mL~>#-BFy>4 zy`%re$zzS`G<2=$&>>3`P!-Z>?}zFUZ|d(on%}s(dV5xN_CFUdG3^XKTIRbDHN(kGz>jwgTn6zkKSMTy>Cu@vXZufoFhslD;PA! z{gh0OWClQcm%2t$vy|_q2=2{8FiE(fSAnb6>>zOAX%QF~gyk=p`IZS0qh_|qO*;q{ z{a-$iOm#bmP)?UZ%QdRDJ>Bl{{2>Lm)w%{@aI{&MASX0QH1t+CoAB^eA zKd(=j*9t_}OOU|Ia+m>{83B@i)u4`mV|1{C3EQQ9>_ah#eP7PhfD#_vAag|MTkjFU zEnu`9_8z-%$-nm@- z-Zavher+neY%lex63ok(_fZE<`TK(5PaF$T9P(FphUfN7?^}K#7x@Ep=b-`5VokJ^ z3r4`*5K_?+4Z)!97*kD-px#mx0w5bEF^aqe zS#edp_Oo0Y>VX-voJ8UrPp{-0iU>^@$Cy8M6f|qkx9@%)z{)*Y-OE_>8%tm2AOK!E z^D}pfuSk>Dq03i(ao=0;=bqLjk-D+Jkdc9xf#MPh4jtPAv&7H!*qF%B|ABx3 z1EpF-;~zyNgn+lVz;U8N<&{TEpMvB8<3eb4cb(KYB}#HLl1f*mz8E8g0%OJLiSi3O z6n%m*c+=4ITe;Oo0%ArR^>FvWHeniC+}SV46=A=;oW62_PknN6VY0WvOm6u{Q0^IO zemJ_%4t94w=4Y0;c`ZWJT4yLfBN~iGv`eRi9&A4HjP6vq1(UjCzoQDm!ZeuX_gvt; zti{EhuG8?F9*5%#V7Ym&sy{Kt-Esl)O??#8K6)Q@ymmajXp914T5s3MDu(4}?poKw z)K5}ojTQ{X7-!Xrj!YM8ctYTi5mvYXgDLFSOb+7F2(zV@nNiV#r++z%k=^z=3;a4NQ~BwUI$ARBCl zQzb?CfUHnJqiG(*Gjs!{O$2+XhZI8Z?V!*eP0N%ygjxCZ2x5`#UcD|C7M%g<2>_@p zxA+r8Z~IXpe@mh6o{t!Kd8(irK`0^!kpdxX4{P@=Lc$sxbe(vpuY4|O|6BD-S!ho~ z5qxevGAavIA_Ud@zL0%;xL!CSt0#!ZABrdf_gG~L2adadS+av!Qjf|5sq;6)Ej80w z-)DU#0U4;$Q<+=NFIAJpabRdE?H}~F0TqL0J2k6br&?>Qq?4tV>C@}ytJ7Mw9lt76 zKutq6YdLj?gL}HU8tto3P*?BvwF9iB$;k;xRGq-VC@{m_k5!eEv;wV8h)>!!qWWs- zVy&(Jg4{};C2fs`EmVrJW@&>djgS_4QlEZ!*heCi;}R93-fWxRXBM@)6v8;oYzZip z63ew`N!(zPKAj4y#qSoMItnkF7ZhnLhSCw$oO{I9nIDl#CVaLy1`}wGg5S2+9e1~2 zi1m4e1mw`EWEXzgeN|v+6sflOp5O-_Q~;d#mpl~7A&Nl1(yq}&i3nj(SwY**G6OYR za}%7)6dtnxeEM7b8*G8Yp#MTvD}BYWReitLI%P6#sS(pSh0>LNfq-eOTp7oup6agE zF_#Lo16Q+55d=ZJ>|i-48XJGn@3#{mo(JJt?<5{lF(BX|jM~o@KrmG9s7l|n%mpe} z<`Tf_Zl3@1&d%Yt;ExpUTODTfpQ($Z=6TC5rN%@ejff_RC~!!)8GMav>S z?6YD8wbz7?-OSgs|5xU5iMA$?M$^FUdJCj6JQq#wqT^4=@w|nMd3FGv#RIgnO#OZ} z{dR`pq-Jru!6o*i=?TJSby45=b{=N3=P}8&=Zdz_+Z9mxa&<~4YnO~euCdOgkBbCsvc_76}R5JK5WRK?e zKJc5qTJoqJRDv1GS%X=F!cD_Q%Eg9|PxZKhgRbnF2YkHB=7R2_l%O#Fx8O>9!0rTw z4u@>PctdRHO6YbZKZjIIQ&sr$1CSwSON*m~5M4eTiJSI&O0-{x3uYi=$r8{TOA;zR-?hD6I6~CH?4)Wm6{BQ@9ShUrmR?$r z2n05cQMzNYM9?b>{{NIOEfBF8;^8x2ef_?5%#70XPNw==&(B-}9mXYW8CMu!PiTwrW1)Zs%ju09fe;*Tb`4vrQW+usj~U z$$Ju`Zl2Fk)$M#TYg7mPJT|0c;Viw}bZj0ICx-HAr3l(}=3*C2JI$aSvaF1cMhc=G z>Lh?ML@pBUhN;^Hh0Ar>Qy50)_Rcho7AV!$daho374)bL5`jHSZMtBO8u}{~ul~BhbK1l2= zB;GKC5hv0JKKsh=c^^e)4+Vq(M)9YBx_&y6PCAWZTad`G*5MyX^fhh(up`#ea6xgi zMA_=7Z3UPk-LGB%q5JN4Zh8J9t@Zt&?0i8v_x}D^eaNtk^E#Y)Y%R^beRheGQOm8^ z4LyVUx12Ok+TwdHrGV+Afycqw_QM(2-GCh`4&_HnflpJrWzY$g%uLAQazYDw3piYQ4{32QyJcesvhbsK#Ah-23BN>d5V(? zU&J&(ru-gI3d(S@XvCH+cZ~bfUXp&(leI?b&|}GBul~R^7g>A8w-Lf-%0p97W|yiA z{|Z(Eu_eKj@JGLeqF4Zs14!QKK#du78%KX4&r55txry8Y-ljQ}x^${77wCya; zL|_Bh^RgGgj3y)66+yI2;;`m_OPSvzs(~9NvrQ;0XsHl%L>unR4D_Utl_6vVs+%tR`nnMkox7#jqCf-S$nC}<>~l%|hF?R515$vkwNxbO;`jP<{XR3S307gT+q1>WHSEy~5oT z{UdM4kxiquY68IQ>0kn2oW~)CKuL5yQ#)YVEtH1c&oVqmt?0a)A&g-`HGUe+E-UBY z_sytCZ@2jI-$ND9Cxq?A%liQGgFX-E-8>3JeXw2)w*43JCa#_j3_!;fuHKrLFt4mB zHuy8HKs32QG_PFV?t^SWGja#|pC}{b(^2N2?-iPHFa&};XI!G(Di@-n((#m6h8y<= z9lHAAUed8Uol2AOtLSWWI=0`i9wp@>bR4`tjX*|r!cp<&Rr^+<+Aer;MRP?HY9>Ll zBN8nZiP7asVjXJsmU;(_z!IdIkGRnoI(OPt?BaRmja%*tK%|}*Y5)2#qFdggcFkA110!#y$ZH5D?!TB#ttpTmKbtFq?7bIm+NV=6j@XX3mziDtVI>t~> zam1h5hV#BHK*|(AabiO&&<dlrHnZ7;V|)xua{7DUO+oz z+G};hkl~JSKbyz+t*)JrwdlF2b}WwK4R5;5}!-_a7nDoM2$L-4O;jmZ* zyhVxv^1!H6E^b^8)`E&`z#Mhww%MeZkBs~T<2XJm&?f+RrJo%ZFoX8y>*{S8pqDhK zWMuNdR+t!08g`UHx#pI0(Ys!JSuFjj*tnrU;luq>xinMzj+@t$WJH7gKG1?XFFdJh z1BY7S%5{+Oe?m%U#UV+@zgq<4NS zdO^tqcMMZ9jr(rxNUX?~D0g@?QE*XI;xmHn1`>-Uvtz6Fiu0t-eTK8M;!}$RabO=7(iN{4@(LVv!dARpA0|GNPs%#y= zHu~N%E4u%)!>#j5n=ayb&j)px*+XV*@x0o=TuWft8O{Dd!dTWDUa=6@?dn*_t@l0i<8n< z7jB8z_f&o>*vXfz;yM%`UjFTZfWGAj7VytyC_H${M!Fe5^cL8A{>!{P%NH1V1s+@) zdy<=fOpEe>Xi-&tjZ5itbkG-Oa$h=udnrA?AsD9fb_s{_`AHhl^q%HsXUPBP)^42_ zB4sw@mYi)+-Fe_uY5G@X_7N26WMnSx{)IMCX%2Wt)tbLU`CVBV(9H7@g^3aGSQXEhu)N6 zjC9_#CCd%gMKj9+$Qb~%*utJ^h?@hT^in$3xb4OD;%0!TxR(M2b|ET3I_Nt4gZ?ju zYd6uB0&Zn|!Jj9yGAz*28Z6^!pZsA}z11X-VC6RhV3@gV#~kF=SS#rwO;lcaK#Ds} z;lydHTfc_5L7*F4T_!4gvejF6Q1RNP-zp284mK`8e!CJUr)D24P+Q&gZ~h*&k~Rn> zn9*j1pzpn(F9DBUQ$g383IZ9nn!wrNzUdz|)8Enop(@?+;wxD6y=XfIMfyU?0s<)( zJiBV>w{8()*I~VOZ?#hW1EPc%7T$Xvb;)yB)lWVDv%uZ=KN)Gptb$L}AWFx@SlN=E z$B4nPmT&`ha?p)Zk@Z1H#UT(TdWmEd%%nv|UJ*gzoKXq&)DIc5fY%du1^M@i_MDWcyF1PVW~CLH1V;FZE|5(^Ox$$J``=P z^e~i>L9J}WX|&jFx*QoH8pD?HMHAb_Pl{ksy(f?Kxn?W~EKvh5b%ow{spR;~ib({o?8AQD?ECNEEZhV7e*X(&OXUp7`SIl}Jz#s6czkGJATpLVdk=cx3#96fiZ#d&aw4-dydqsmg7* za>4J=L+N7(P4H7M0RUKM(k;!0k>&n@XZhds77_nq@0yf?VRuRXa9bcp3?tgpL!&vy z)72Bv<6^iX^`qGtC}+ME)N;kc)%JcFXp_&d858+6l!lGaiOT&6u93UMXrOZ570oWv zCA7eDa+*YMN%{;H*y3589|{n2U|G|@*iQXna`MreUx(tGR>{{h%znt(nb4cXxS4jx zcLaUl-hlwV7RUlBpx}}0!E{p3-W2%1T&V|Xv3Qy=u)b)}c7velJYe}67{{6VNL2Xm z+G*eTSUh)%uk<%FE`Bvk)1aW9MYH^=wA)QrfYQHQ;u)YL0vC-mZ94bq>Iev^SXn=*34)8Yb54hv*9@4{e$B7PP6zRi}zRwwdu86HsXsZl(D^qtG&%)4^CZ-~;JD zz-@JCy=$vNEZi$&Vm`sOecEj0 z!9DD7v9xJ?uhoYHHX#Q*5$4L4C=lFqPqv!m|21Vmwvn0 zlgua-MPrzrmQzJ%MkW69HVtmP(KI*z**QH0_VmN5r6SBV`+!7#z>Ze4ZviTJ2A$VC zmUpna8W~r`77Dz+Y2x7~UV6Gcmaml86S-@w=a|DTr_e+rW#2&Kj7wfqp(r)6`j|BM zNyv99#c(ybTSfUzEaUcItUmUq_T=AEUeN^2x2NOQ2MGw&_-W(~Q6f}Jr$GlMTFqZSGmYuEq8ql*x%v5bXGs38~99ocmKG*u0?)Its$( ziA2Br!lU(Q3$GVR;d)2@`c#6a3-rCSIO@?BfDR3zUh5qB6dwuu72K>L zj&2RGtt=C6q10(SkmapZ)Buz8ck8Wt7nO zc$ZH23HSnBl;eoikwT>t&#@eUyBQU2rw!f+HX>eQz%L$Bp zHB-e`s^SfV;Q;W~*)3|*W|S>puFA+!YP76(!VE$+D%f-A*?qkoZ|2NAEL>r)0@?S( zO9Cv_VJ2%!&1L?uxgcpk#(TntZ7+#z6)lu`Mo=+lFZIsuCx`-4Z)hlRxLU}wpU04^ zI6rD8xWr+wZdbU0yM;I^DMOJ!gGn4x`zkeF#0cz|6xK5tnZzYHauIyrj<#YAVDsZGn)CX)F)IpE9{g z+tZYCRf{sXm1*^Ae325xYd4@G+YQmoF00fxXBwv;S>Diux;`as>{GyA==@oIwn1Izv!8p0-d_LG`aKPVTDJ$OC3vG_+?q5ShOq$AI>`lFbgOYg; zR+@r|(GJ6Nx8l1Oe_OPC0yd!*sz#S(T*b#P3zrtPB7Z#etQybz6v`0*l|=4Jb|uOm z>A(u|!`)*Pdhygo|J3bwB%odNw|fu45gC6X55SRgKoyht?qhcU*|`l08UR;Il8)!X z?EOM}viJq!x@kW?g?X%_upzXbi(`XduD<*7HQs`Jpzm%w?9DO{g$p9%I{fA9;P*E* z?7kznhtY!|x9>&G(H{tp*4GiQAbPz*w9{qT%6vO|iZA?>qoD)#a+-n-c^2ldAJ5$V z7z_Vjv+S(72b|;Bldi-H#hkF@Z|b2UHS{LF(H>2$)cW zcRAV(ry0nfMQ>O>mO!vnJP5FOd&8P78!A(uI-uwvic(?nR%^08v>U)|o0vX&%7!TE zqn-8fe!|29spFQ>4a)pY(fcaH`#PTNkbwz7vZ^i7ObGT@$o0Ty;CJdNc>v>J7o`7s zjK{8>ORA;9Z18CA0^@?K9Y7@5Ns1;$1BPsq7G+O+E)S-naOA2ZVaSLaX4;^4Dh_RT z9JO~wxjEuM%L~9428Wr@A8}OhSu!N*^69CGWl&s_z6NU5cE>)-tfYUDFyprLD|XSn!|f%Pi!u@qFEr`yS^5# z?a-6C$`N7!SI#tj_=s+CTOO53YCjh|_}gQ_;w8fu>e|Oq?LZ!F+GLWKJu1(x4*yN! z4tT%m*6{s#SoAA8j1>p#`{8?i1NjPqQCja!5Q5kWkN1`f;-9$%vjQ^JmoZ?94kh`s zeSH+Dm=@vL%YWZn3@}eehuM44^6v1IZ@SEf2uC~ALPBwzXHXLMq{Gij-bk(gEE&E~ zyZ#kVa2J8aw%oD^yX4j2k*|_jwi9QdN~&QBhtNe0aPy>p`mVx8C$s}J`6eSJK}d)fzfuEH zi1!oX-PE-JFJtQ)r2f?Y@V_IP51S8g(Kh% z{Hk{Us}W)K*PkVy2MipBJ@ABlhU43THAfdB8~Pfr3H z66Xh>`6?UX)T}^M&{Ix6MNkQ%xPL@7t?j{<=Si;smm~DfK}Nzcf9JGd){Ip;cK_K_ zOqkEjeQ#7Ch~qlHpm56N2FfI;1IZ zw6)Gbc!NC}(Zs^y`nBsf{9E1S@4#Y_n+ylZm!l6p!-Uo=U=Pu=O0Lu)ea0Z?iZ(ox zz52V3OI*=&K5#8lvJvG*?Pxb%j$qJ~{~0vVu>NPJ|05pAU2kaie-x1r{`^nYV*%*r zEWjKfyR!we7mWFLK~&- z=a1C-$RkVqO`ZBB!KNyL`9A#oH8n{ZTwrlO#el#VgkH2@RW0{OV9@B2;UcvYU!h%* zCH(h_F4I;*e6w!c_oBe%HKYGYcGPQTA9q3${;3!o6s#sW|6}|9WBdN&8rQR&|Jc4w z47mS9PX8~)sG0Sa7yK}X{KJ+EV1s5kV#%Wo6$C#N5k5C5%MwHw(}RA)-v~_ z4kv@n)by}FRrkMn?TM9XY`{}tvnQMbrQ(NbJg=X@)A&fAA^&=&Rs{bHc7(PMr9b@I zj}pkPSuy;9){r#%!2kDGx>$incYFh*Vqn15llQ3@z58fvJdu1M%v9FAJ`VBe9zPh} z{LX0d4_u6qmVg~jd8dVzpIR8NWxHIcc7_SrXl02{^VXY7@~`Kx#3JW@NUnz1OLY;idAUOQfhR$3 zc{0g(t!*4nn7%Fl5?<(jkOTp;cm*IL$EwJ&ayU_Jpf7LE6yMSfUKOoy#Wp)M3yw+f z+TrwdndE(1M$bjg24E@}_{k~RWTl~inm`btO^@%>HX3e7>IMLxcXv(Xq(Q`Vr4<2; z`y0=jL7`Kg=$SL}g4M!=d>8P#|Gu;0UPi&2T8zcRSLlQdf%aAbeJQwGS^l1GOfmTi zlmKiaj$J=rm2D;`!&l?o+d-Hbb9hMNK9OcZ5aft*+#=aC;kF>s9?f092@w{C3P$8e z)0GB))>k1()RR8N5posd*Q9-vQt!q&MI+#`8^eZ3Uc~VFOS|0mMnEAr0-7pW3{Q7H zr5li*vm{8_5-MS`$u#4$UW_Ap=`8*;3heN~74Z}B_t_6g`VdWV8(b9k1ub>)ZU0N` zi6mRYb&m69fBqQsgPK{C_a0ESUex}V6UaUWzXQ$ku~#&>jzkHDFK4rww2o zn_3{ktkbiyY_}Cm82)|s9^>vF3dvLcl;psYNp*j42J~T**g@?Rj-R7Ot)iefMh&!Z zjmB>eN-rary%>dk?R=)(UuYQ>@`iqd|6icN?zwq2g9eAzmB2*D zXJahkqDHlB_%RoTns4uRN^MTx13u-q^W}`>6B5kOmL8bPT_ockg2K{kt<%P7UwQni z3LdG)SCKza|W*X+~Gx?f@C8a&n^*P53I$0iY; zkK8gBm#;Bi@-d`vi#?SMbQoXsVS6K?)C6LNY6R}_QF!REAAW+mBI197UrF;bKf|fp zS=L;dZW;eN$wslm^V*{Fsm83T7tfQoIhQicWQ%$2;D2#rfZ9R=BPK z5CilG<7i`|MB|T7zhN^F{aySa;cq3mhTs5Zr{g>Ef~W$UT2<4HWB*IOPkIv}w=Yd1 zp?fXq9`N~!G^(0^MDgZPLEE(zyUEq&fFBNnYBeieVx3&P$6U`TMB>jwCJv_%!j{CM z5wjWmFE{#^enY$G9!N66FsH|{O^KZG_ z@zbpD>3GC}Fu|BHGm_kG4~;DPV^JAhPRsmPm>sX8_v>N_ssA<^8+3bD{Q{PwCbxe{WfEm8vWbmdULOXG2Q!9VAl-c*v_^$> zc#ZmovmP0w`J0*w$19RLNaGJz{yRd%uj$X^>M@Fp(&;m?G!?Shb_+dcVuz<4*#0OO z97qL9*P$-(l0cW51q0mhvXA9&P8|#Y9<#EO^Hqrwrdo zK9AZw^NK%h1!Q)w%}vq9_m(3RTAl>48cjJS$p7 z`B42oa6hWJB0`5~Sb~jUDsH{5yZ%P<>|G>fo(t67X-{pNPcu$ng{fMvg$rJ*|M_fC zanAS!@(>>Qm^5K+T(|r4*>T?e z&8>@-LCn6hPYjZpEbzX7poH@5$T)Hdgme=eC1UP8lXoEhQ#2>8sSN0VdyqIX)fjiU zX!!C?*nYI}GzCnNq3dDkm0{tkif3}++BdGXT#df;Usvm|df0|o{r~HszcGn%hbMw@ z1ro)}?{CHTx;xKx8np?UbXSc?G)im?2Z}Hok14BWr}WTGM*bX<1<4T;HT)0b zp=CDHx5}CR?jvkvu{<2Z$%3^oV zat3wQU>z4Ujd2JUkz>_hlr4O21wH=O4GKhwC}LOLd58!I!xSAplEg;x2}L`$$!Cco z0O!r$k#G`{2t+@02W6kxVvnz16W(23eZ!g$o6R0iFKwVOE>agw9#k9LWHUle!Yb{h zuL{A>t-KtXF7jVDN%+7KSsX}DUw7a;LGHCf9$?XBaRTt0(rekoJQY1oh2O5CZ@GNu zcSAn3Z|^zAr~Cxlk0ZRt?Tk(}OFZ3pwJjJ&80f2)RA&f>Q$9le?ux1*=_6F_Zc?>f zU$;US$Z*2VgGk;V!vs7wZXsAT`~`8z?`~zuk;z3mo@&H0S+u+N@2Y5h`9bt%pu_rr z;kt8yAS34~j2ZF2*=WMdZVbcsJ>j%o;)JvA1ryO z&s{!c{e@46`*&&6z%KpXYkzc+^lwLd4mKHZC@OjGK8Jd)XkRM9YdYL)7((Qv$j^c3 z_A0R1Th-rqQyCZN6PmKlnh{HN%<6Jh+4g1jZ2{e<7_4WG4Q`t2dNh72hIga*SC!uh z!lH9~3D;6I&sg%=-S8*YF+n(bXjPeulRc+B_dbJ4ZjgJqkZ%2HiD4n3GY=EY)$YDH z5W%=wQOSw98%TaRt0d%x(#6xhN`0KH8TnspH44H=8f|V^Jm)J>k`j)tyNFHwx9Lfp zl&8SRQYtp`-gN1>&WM{=8P=BkS&i72N2FIkHWbOf5h7WzRT{8%I&=<@!ax-SWOQtE zpklhGMMj#jbbc_l*75v>YJQ37J9eM;4dyuB58bqMoQUPHude)lCrKTtp#k_L;R!Ysd>r9^dN* z$Q~GNP%H*cR7dOI3gfj1)_ExJ4j+cJb@0jsU)7^#=oj>q@y2)%e$r0yM?}bK4o}!p zcQVgWj>Zl>e)6zn_0E^_eLKT*Jfb)D0KwIqXcw;dEh+q3Vp3t<-ag9V$JD{yB*n4` zWCPGHlsx3Npp!r`t)TG@YxGrQ+KMcC3Q7{vYD=K!sJ4BzT4zdw^xY$o>zI*uH5u$Ol}`NbgBuLf^erd>?#c%OEI|mhyEZ!Ss9-9UUDK4pU9K)Vk`gm zkjO8!;6IZDc!QwqB5rr-Vt`adOQma8c(zk#yg(sm@y*!_ZG}yQb2^F7=B#g|Gnkpv z9L#^7Mwn&{gr5#mD!Dr_s8v9#g99|a$KO2eDon`(ay}#&IutibMlgaxq8eX1MK%>R z-_4LPbHqM&81JKXcGdI9+JzR%XeFpATMCT1?J?fr0;O5yv%GwT_QwCx3<3f#k)J z_1lhuYGe~J@fPQCALAQ#=%}7Q6P-ZnPsK*j<^FT5{diNFq*(D0Rd-9{QfTMXgq_EoYPE z4|(w}HqVvB!DYYszcW<7^4Cwa=E;77vAvhYGKG4NsSaVzJ(GIpy}v zvbeeNW@%h^qghOB)Bv7yM0}v)wk7)7ojhHRmmRsA2iJQ6D5j_q`hv38r-Q{S0cpg=U=->08j+Pn|6UQawno1(@4}>y!B0A?l>j zL_XdG5>}Ck9`MFAR^fC9e|$g;5m7 zBA((8@Br6rZ|opS-n(oNpYLxM@P=LafS{~2Q%vdL!$k2f#+zi5?4m&C^{%>!-d$x9 z&UTF}*So#-9e=#Rc5ZsOiYKNKHx*Rh(WSe3#*z5GQJ@75BmvZ<=fRMxCfjookJfmN zsL9a@X`1kmS3H@sU2J5u0oC918!QRJgkS9jXtS6|f3*pjCxT4h-&W}EUjEefmiql| z*@R5K2d%nGu4GCn+;P_IH8C`ZLwg$N_D8*iMOGofM3;FVq_l0*zjOZarj?M#A2lN? zH2a4^X08;6%dRqBjJMJ)`uKzR*dwqc+(JhcnjjxZOz>$IB@9Rf9hNdFtb%fZWNyKQ zX0wWz9b%d}9!{GUUImc=F$=lK&l7$Y;`8psPQ$(Q#WOQ3&TrR&6vQU(?3p`6IDti@ zZXHv81$4|%?!+pX02Ln0SD76Z!et)$8ZuVv$IQTeF^(|6Z}!;~>l3vF^Ohpjqe9`i z?Wf|mPZIeF^B0HCS)!WIJI{`6?>!9Px)n7?S@F3K^XPT!@4fquo4;i2@k>QGQ4&?u zHddq>p7P+b>z_o1T@o|4)|t6vXs}z~kcp@#YhZ_ZmDyDgcsEsd<~nf>`(Qo(rPjIy ze}2^ZS+HYm50%Z2r<43oHpwwNvf3?2liB_}6GszqYhWsKMqOaBnxT^@%$LFWpB7QB zJq-ab?GO1J;)v7`3L=?xg$u^~nNzUX^HPf&uJ<~xlG4U2KAzb-?~u8ISxC0?so=bLE)KVejJ9h}u!&i2{AVUJTutr9_sEz-9W#;PqQT zh&SM9W4;Y0p+VoM>*}QdPv*Ra=o7e2Z;_1xFOny!e@vL#IiqejGk?FqjC7T%^TgCf z2iYw3nnltn)R+z1s=WRlwD2CvKPT`evXzFBnT4ut%ro;okN#%=rVoZ~SkldN<3%d@ z?UwpSPGVQqTG|@$t@LPKqs@|O)ftPAVAPL2~P`M zdg6lQ8RZ=m)R%_-yciQ%jf|Co*3y&%6U!{B48vtKsPjB;Xv5iiAVNaQ>X!(Y%dcxC zs&UPLWt{WdKQE)PBuTe9XO(~#{VcRY{g0tKr}{HnY)$o-zO-2Fku;(Md#qwK*;8c< zz4MNwUQ{FnU!A{WeLWDOu$RA;Gc)OvfBgtBTBn`!_1O3E(3c$94cS^dfgBs@LFfcB)rDu+mi8glrvsv#F zAzqZo`B!*-K%``9itoPPOg_$+Lns2G?H!WS4k8#{Sh2SaIYj#;Z-iR!?6YG*nu&R` zEoQaxPChYQ;q-(1bA5;-BTPcv16Tv6sKOXfrjWFE zzt)~l{1#x_b@oGbR>#NTKeG8}`q6{rjY&K&3m6@-jGqcUiN%RZhkYWB9FIHl{-pBYNX0REdtV5GOvr|a*hH?T8}(QOSTT)~N; z4aM!xDM`M=P%rxQ@DL?Lo)u=tjkC=MuAAK(Audht+?glD(&`Wwv!(ElQ`=Zw{D<8X zGZ}g;PjJ(%fAoX^6`rpb2#qw9CVQz>0vZ_7AO_ zCuQ?0%TtkLExA`%WXK83)UbUt7SkoJ{dGv`xO&wZPX8Q-VUKOpB2U0y+upY!5f|CR z*gB=xGW#x6WLPWZe|Q4BGUj@CD*4uw@DD$t(g`b%NdPr_W^!{6<|KTeFwR4M27Sfu zYVsfwFXlIQx8VBbGUxwIy!qn`G-diDk$;VTiR7fjP^1ol(60t{?8G2dRR?{z>+mrP zAmrocFrIdzGV0s?(bx6RqMvd+n_sP*Yv5~^?i-?>mTL@vwMZ!q@q zpNR1l>R8AW???6u;)qbmq;2ait`_Xh@xeY`^mgYojV1(8m~H*P*n9J6s=x1lyu6xJ zq@+QnN{CD$^OPhqWh!KfGE>IP-B75gOqa}Lo->p>b>PRKlk%rn3JEK0e0zdrBZ z`mXg|>-+hqrF)lVf?CvkPW+P98Y2k9uh zWaIN&zYi#&2&%H2>!|tuUC@YAWsQAlkP?yzT4WYvCK(FcO~w`kB?EdF#7mZ#xaxv&`7Dqgkg%03Nk{es=8TKLD}s}AEs1I0R_`3retf~Yqm9ygk}U7_pi=DKnN!PfEAyB1}O*qZ9mPbYLAYCxvEen8N+j4 zQfn5AR}vG^x}AB7C>W~vLu*nGmZs0!u(GJ`Gj|6D3{{4ZNDi<_h$_%`%0JU64mp() ztYTJAB^RX6TbPEPGcf4>T_n~OtKlf)>_lfF! zJ<)5)y_40MW~vMe%}5IycvkEs*I6gx4544!oju%>`%@)-7?g~{VfNwUF#2_6lwr z;cHAs%LfL7L=5S&Arro&hOh(EYx&_JF#T?`$qrZ9W&(KX3Q$E==-(=D1~T` z1Tf81-1L+x+h59=?z3w5pl$&f{2^n7x&eQ<-N^)i$oG*bz*xim&O3TIT_+zFRcEwn z++7Gl7`-049jrcuysru-_WEZC5wIK72xp+Ptwj-$>K_d9ZoR0mdZy_?WLP%)nf~kZ zz>q}5J~FG4{Tg20=^teTFO+6S)#M6!ZVhbhs5W)f&}v*Ja3_g5hO%EHIT&iljnmX{ zS&p??2;gqd%E~SZxVh*2Xy=v(-bK^BI*P^bD>PkC!cL5XE(K=3BbxQHaPuQ%|Yu7dbNK=ZW>WuF!{FK6Xl+~?` z$Ex|L-v&qe{wo9}S;F^l$~<#_HIXeVSB(Bnki zuxJQUq`NmDU@h`=3Aw0cF0*I#5L(AutjBLdyW-&64NyV#ZlN~$I?DWfKMi7Y&fN+- zfg=J^>Ml0BCgG(3<)YTU{R4Wmwt3$fR20zAXve&1UZJkm}^awV3`XrCo|B= z4Nv?{I_R00`qyl$QtO3aU&L#rH#N`KX&z*~O2Ka#zm)kO zzxsVj0>wZP+H*`j4N&MTN};Ra%nyo6BdvnYN(j-f+qJ(fiUzr=XQdr}7KCABZBu?@ z&EMbqeZT%3kO;OTrs=JhG9A2J>51~IA1@d1gE^M7;-&xj{3de?$$?4q1U@O>%vbz3 zQ?e9pMd)*yIlsPQ42egnzE*Y!BTpA+aobalJ9+=VXyL#n)jo4=-Q zIl*-3-fILov_Dpkf(Gp*>}$E!H3j*7|KOmIGrmQe=mD#?yOn@+d;)5OXaP^1I_6^X z$3WnjLUt~uEF@iY-6%g+M}Smnkn5@wM}GSZPmcS?ECkGfV`5q8yf++vH$Z{7>iS#% zGU?Y8bTkr8Q=v+zq6M7+#ZvWF?VT_Hp}5|7?7wc`eoR9SNiv(9lx4!Q8K7>c{ub%| zZLnVl+uB@yea!^7$13-7;P0nFyqPasn}U9bNi?GI?X;7A+M{ce5O@ZZ5FTuKu|b^f z3E03_!*AZWtq)xw1Y?4hu9d7bd8?7Db@VSqijtV#fI8G0ruNgmxP%niWE0!7W=$yp ztLT1sznUXhKaBwp+=wn6q7A&V73bf?0srK%wI#2A#g#p0l+tL-EGavehVq#*nO&tm9dm#*SlfhL!qh(~`;5c$oM1-#W;f{4%ux@}2 z>@Pr}jCy(5_f|Hl9*Vdiz*Ix7!)rXxMRdP=ve-Q&Lj=&Eh2f zEiHBA)ytlr3XO}4tA8ILbx%MNh{-9~vKE|W3Vd5#Eq#QHENJBE&n*0Znruif4lggS zMn|DD+tq45lnH2GFB`tt)#Sp7!yLA=x0~`E9R3nr+Q0{&_OH@%`1&ARO)~)%S%&-R z-0Dxib^gB|$C+o%c`#VF#aiGYVeiIY}O?lpsq(%w*s;;ZO9&|hq)~8xdk@V; zB7mL1`L27ZYi&#la@0u?cc<2DODOe?bU^-tvGkC3<_pSO)}H?*VFJS2ppF~4wT%s( zOu|{*KTtc;Cs5t4oMoP--`x-xA93qiJIN(lflA;oVbu$k*v-*{`5Ya z4!d1^)|MM_oayY@1NdY-=n|QRwGY=3qZ+A0c>D=Ay(o6Px~1jVb9HxcIO`D@Zb^XL z>N#9@LH+h^rpNB?4H3$QYg*?QjSqrtCF_(0<)t>fk(HVW%GW2~?kIe^IDJCkPxkQN zB`z-Rgh*U>=9-_ojs~bRpp8?d)1l(}I?{fB3nYAdKfE|RKR^Fbv4R$gK(FNzSLg6y z52aLvXiz}cMr%lbVh?P_8M%awRtBQ)mzS5Bn3$@6Y#UPK?fM(THOQ%sq`Hs4D{FP~T_2YpHqi(O(B!wQ=O$%fnO zlMO#N=^bF$2q(;@))x0`veIl2M~{^d&)p^o`j)f#-I_$AWN>%&dFhe3ZLq535D41% z$vQ0ZMfv?yXv?uG@H9DbH2T~hz^>@$Z$pJ17NY*y87r5nw2_CzkZ{b$0-Dt{&97Ms zgec&^xuBu09)f@Y1ma=cDUu3mKL{sah69>Rwj%&^dB1fn!3Gu*K%i`MiY;q!#C24C zyJtHT{bM649T*z2={O#YOAw7<@`ZOst!6Dl@>X{%I&TZtT9WQ3VF~(?82U-R!&nYZ z9p#YBOm?`y%1+rq0(pu~uy|dyeYjQ$sz>>hhnE_+50SNK{e+FO7J=M8c@W0-OZtAk zmX#4HWy=J8bDnNP)RhZFW2p1|29eAv&?NfVTWEV#1hkH{*Q?7=*hpzCV*>kFqbOZ! z8yl&g%Fc_Pek!01R15r7;#+z*3N%w4t1crX&+46LihaeOA?4(F!A*ZIM=J7ev^W3nX%|?1)Sp2JZKrvR_R+6QG zfdO1V|H+~Pz#Jn!t>e7~1=ZUyc5B-m?J97fNz@*jddIqe5IYp5?a6z!dBJqJ?-2Aw z%Y~_b8Q1!Q1)m~*2t&oW4YGvwz3bHC_i)3i=YOp-PgxR9bTj~Ifl<>Qzl_WE_&s7v;0jUH&?IU zDdmMIoC`(Lzu-?B*N2LSkV5LAjPDrojeAvxL`5ZQ8`g3P*p@#GBj`Eo*@B%)sDtpe z%)7LM8{hr&6I3c<$J(7AT(x8WkYp4s5;4$K(%TU6SvTS8P~g6Y>u;s~zIzxLgXCey zk9%R8OtlH7dLuX)DhXkGfC9N+$CEyc<%Iu9mJoG#=KfA~T}IZrgp% z=6R(+>_WEz9P^U|KmPz<1yaKKThrFybbq`j2C;R@b(yh~>c_)9U}>D5v6Wi~xnDn@ zdhk_Z8gk@TWB3~uitIrM0uXVZy}lV{RuZgQPE--r$mrT0-U0r6$E()57UNj^{Kp;$ z}5HHY<-TgN?r zpauVjgFHo|czl%>|J}@8klcJKO7FisuaF?KZNTt1S8ESbAotr#cf^wZ7r%w`$`;P6@jDs+HdplryZ#<5U7ON>i z-DR*=$M=8sbJXv#eC(vFpu7zw#UDa#E)rpn;jp>%tH{s3jS7=)p*&4$-09!;>0gPkewU8 zTl*vhQHgvSOPi>~Adn6#y_Svc*{}5q6u>jv_q|<@dsz3nwb=gem37xW0EZ{aZ~Hle zCIX|pxn51`1jScWNI~lVwW@yX(f?Xi*vzX9`icU%WSbK)ys0JrJIno{KmHAu{_&G_ zvl386#6+~^zdZg{6QoD~Ca0T2p|S(Z)5-+a}u1y*2SN+?PK=%j>`H?7D4Q&ogUxEdHC{LO>GGd!D{& z(1HB7bK$SF`Zdomn7ON7fdAiYEv8p*p+s9DR0XsB8UYzyr(;I1JDRkOH+v(h7n~md z@BJ%(z37jJQRSfQOx0Cv+$=ZM(f=Jgc05YbTBurV%%42t4{A?kELS^vew zi$<=;4I6LWf_2s7li*#6*0n>_Ot7sjBzJ^SmWzf7Su1w(<w-HY9 zo%E$}cbrjnUsr^hQS8AR;-^7~}|*SX)IIM#2?CR;%9f84o;!V9+jLgZW> zUJgr*4!pnLiujEY1x!J9KX7vX)tzC5XKEhHoW#DKKQLa{gmVij2h{2Kq0CLrg?Eti zA{^41`fC6Z1xL$2N7S&3{}$9Cc!Lt$iu=ospkk5J-tPY3CP#HD1OU8l9vH-`U{*Kw zsV7VWNkwB{XFtp$j67w>CaUuLAOSRRMLO)^>aE5N3=^UPs?wt8`X+Ah_f1%T*%`}3 zLd;;tULJG3|2HAlLO}cpNdU0Hw%!mV3N+=HE{Xl?A9?^QGCIMXcK2E>fVIsK;DXTx zIF$YtQxdod2O=#CtuyumQJ@S+E5=9B^S3kN5(0oXyfTFS8G#M=TQ@Gf4#Dqdgg5@g z@Bh9(5lA7Fv(Y_&Bdiwy)T6!<%51{ya!9KjT2C43HRAmKhVN5Cc>yhC?x$@MBEV7r zS7^REQzU^}mPTs8gkt(*U3{UQ}%YFelCzc=*4#(TdX zGJlX76^)1(SyJyGpZ~j8Z%ajdB(a7cE!6({B5VU*0R?h_SM>vS{#w6AY?RSLd?ci{ z%|^ES7oUVd#=zA&#~*tt3EOPf&!H4a(9kIkVHd9dH1i99{2`pW%l0>&v^g=KB4Ad9 z)CTOV_@8F(g50hj9MfBC--AWpZT@Nj3Iy~Yua)q(M9StrCqR6=x~_VX|1?RgY%HJv zVxm~4^fyOibIMcb!OWvEah(5M5M-@5)PFbg|7!(YI_GV;2On{UQR}BW$Ds``?q#1d zxbJ-#_g(g7zMX>iK{ZCazazh;Y~x<&Ujs@fPfrQW=ZAp;N}?)L`V$J)L{t% z>LOpcJ3H>1PXf44S(TO>fqQrszZJ7Vl&St*6gDI!E!eZN1)-^1u+bgQFEikZK85Ol%3mlXVa-+f&7AQ*f@gnl&WQ`Z1I$xmet?}I)qn*$B59C9iTG$s}Hp3y&U477; z+UD+i<+iguK8qT2W5BihM3c1bJ-_shP|7J?zwzAluJj~3!&#VI%eU9q zIN}C!wM4_sdZVU-PXt7_6K^+t&abYTWu_}4&p-3ow1XvhPJDJcGoQ4iq$E;BA2__@ zSo>_r_bE}Kc9PpNU5BO(8h7u`Mr5eYWHmagt;{!`*QVc^H}*sp<$w6b)a0a- zhi&)Cj#t4;U*$?WoiWGk@}O6EVS1x#%eMh;QR#1$+@l6&Z$?cI-Yuq6-49)&`2*9s zK&bCmDNYsOs%DLs4eDb!5pc&R|uZdj|RNr1Q+UX8DFp}o0AK+Q6 z@vt{ryh@sDd=Ouvu%?WkT-(}TPUKiNdLsh~hywqJo!xa4tC2@>5p~9qQzUl;vck8g z>aR-izUl0@#H=jWC)pD*WPeTV${!CB@WQeDd`D!qUxc4H!?@i(KEL}kC!$ubrR&wF zj1Dft(Uyp-w(zM@^2W848^W`dmU%2wUICu?=f;Zr`*mgu59bj;Ul*U-!2X_zmkRmH z?j&_?n;vZ^e;!4CN{WDU*?VzDv#aFEaVr@a>Lkh|zC5m!Z#>-jY77CAf80t8X*%9m z_A&8Pb;jEj<5ATwx8f&OJRE4`9#GcJo!LCI)|?Hp)Xhm@iiy4T?*8ByYhe zGdJWJ^)8BgTX6pT$4h!ET)&zuG*SBVo7JZU7f_yV{D`09M;js+8&r1|k38bG*%U zVAcId+2A$S6|eZ}b1WTkovRwHd@QAYW1_j^@v&hZMy{=RNpte=j&hkr6J0Oujc!O# z?z6{-@&^XZGLuWi_XY0Y!d$2=jN+~)`4m4ooIN_%<<2e1iY~doab*;e)0z!n)Lwwf{@h9&IoF=SxI}ZvVx-b5{ z*XfhQO&mkG;j%R*0d1P+e zc-`~~^eSo4i`u$%o0!L2=n#GaG>?yAONN!?{MT1i7LJ@jCA$>T<||WgzUMhfYCqdy zJu>1zkf%=8TG_FY(K|n&V<(d zQ$c;FE6yOl$Sg?*6N`2vg_?X`@9D0JlNffg?TDe#v^S<01Lyq0la{jk&v+O$~+Y+7p!0#S{{+pN4kf&w%M( zueeR{_Gw4-s@Y9>_q_z~MFzZv9^FSKJr9lA&wkVta2ubc*)ZnVYzGLCkKmD~lZ+NuW%EVbq5Q*7j$Q*WMZe%MZ7FBk*P&Uq?v z^p{3n<4<|Z^ZIj>wNfgw=AhFeZT4}0x%ba_FNsNyz?6MtyP|b4@Ne!(-ZP}i{K1IQ z0udGMDfez2Efm7%ya0qcJdoG9GkdB*`R44mjxmXkDVidF+?I(^^lDe_m*=!+f@bFW zg5L+(jGPuBvx=xoQsa+`rPSNbwiFn;GP(3}Mr$d4Fde45G~pf8g^l-I2q8G}ut0XB z=2>Muo$YQX(cv8)S>{7Z)MvFTNjQhoYU8H`9e0$0)u~#ons}YFMl?Q<%bZr{{j{q> zMYU)O_;+WHhZ8_?0 z&Up8;cCg*|;~~Sar5p_KF3C>}cbPk>-MdP`jiTIGyEkZs{!DOuX@MX9y=!tiDo{iU| zE!XDJ1C|?)ln{t^O8q4=hq<1^R^JMC_CUXR6GsN(#7nmAx&k7t`t`9rl3VhZ=Z9KF z`=5~r>~QRl&0Bt^C<0U(UqP__W(ENPvt`!%|F> zoOetdsJb#kxW{pT_e;q!W3!?ySH7xYuI1Uj(RqE!KHp*Pz1rs)i$y-x#}VH5C+&}z zEfaszuL*ND8~Lm_4<7iP5b4o}AQg|=a~G}`(Q&!*+t+*(iec5v7*eCRmy2$-WEy8aXt-CKf*~7H(vuIFWeTx>U^Cg@ zv15nkv7p<9xMy!wpqunQinQk2^A9(rgomhQ8nt%!zA$FER@Lij&px=sk`W!+R>by} z$f9Q1I@pbV-j3x?=Y^RdGW}W+yY^-9gDpp!=kw~mIN5s)ge`#D2)u4LRc|t0MxAXq zu;|ftjQsAGt$zFtdEALv#U+v?BK+a=q)jg`AD)@;(oG%%7JAihDYzI$^v@JGqkYXn zM~w(MX}cD0YdiSxWO;M|MVZB3z9&M4)b`P^k+HMG!PCI1iC zJZZ$J%~H@XbdTzvpy5DKN);|vR^OZSIxTzQpRPTJ8PZ00xpM3!J-7yGW|m$C%Pv)< zVl$JvWA4p)IGn+D_WbzlpxfKHq`cTy$_Wm)?(2kpmDHRRALRol%plGI zM96mG)+ZgR2TBV}gw3A#+EgaWE&W#L!2=@}w@lucwwgC`_PqC<8Sd*#G>km|M3X9q zyWMU^<>OM+LYQCw@MALGPs|Q4`4YNZ`6UT?Oe&l@%`Arpk}&)EZ+(5^*F8MV(m{i9 zIz>Xb<=gP#q2O7a<*-~$oI~!oAyDx+KhgHp_X7*@d*hRYhfKxaRxlHmCM;1}pGth;+U$<2&p{R4!)_Lu>E}9-hsl9Ep66R< zD*RXymIk#cTZH*n;JT%mcvBHEe+}Oql_Y0eG0s~iHW?2)jEwJZX<+W}qsTzt_Sd|B zP9gYRm)Lc7d9}}`;?sVQhEI;ANW3_wJIZ)xWo(FuPC@*$@RzJM&92P)KB~_IP5gm} z#kXLX<^&m=EsM`;aSTIB2$fk+jVLcoW(qg;K$Ogc;q@0BjFXy-d*EwE6?SjA zfC9&z+*EfDTlgCBFB$f*tDZBY`gUya$YfA=COKgRZn_;>lFF$I3>n7ZnS8QFjfbSY z%NJdj9_)UVAHo-7uibAkZ_if1phn0~ZODRBz&K@)5{}?O@Be%yCtQ1rS512gqV#IQ*;x^q_zgj|- zMNNKZlK1JfSP`oCV3w(6o)zoNP_h%G!;fj{EPh0&hM#Lc7!y@q7bBandkd8pC3I>V zOUNO-^=&s1ZE5tC@OR=|__eK>9tvVdxGm!Q*a$236Y`Cv-Ap|Trv8k-&dd7-hX|Dj z2q~WLKY3A$Mf_8X-pv>|sIRpQ*n}#+x}{*_g_o2w?xsv1XVOzXvQQ#UFk|QOpfyUN z^yBUe&Gv1Rd{)pPPis-}5owZwixq@FZp4j*E7(=rL@OlLxXIcRsqh@5RII6rSJ);N zHTXbb$61X`%J9gU=eWI2%Lb!hs3@o;eb(s{;9Pmb{N=LO^TmV)=u_2R6({TI|uH?qr%Ve(CBZZb{6{EbS& z4ED?gB$;mRb1Nglaa;p-+!bN`NxndXns#ztn`yJSca%eir`dRS+Kn#AZ=F2W z1z|vL8xdeKBPGW1H$qJakvC!kGx{j{mhMdk9{{vuE_qjIA-&TJ>wWVKN7wp7Fk z88A|+=5HR|n~)LRr(bBieW}(iY~~j4)r83+Y%b4J=+tI9-ludY0vpT0X98`w-9k(v z)Yts;(Z&)u4Wc0sV32C)jm$!Q1|uTqj`(&`W_d~;LsvS%0+NZtT`wUR5+>+qa5!)= zCKPpuUP+6mE?J|f+nsxaK(_uw7xUt|x~FAmX3a%au}_CsH7v%v%SN2O8l2}$Atkq) zthqsZ!X-wb``nyL*R^&tats`S2(&xv#hax)bZF$nU}w^b#k*;C-Ts)0Mf1%tjqS3> z-;VZ_wz#pN346-=7iPhT^#u!uH&tVbg$KHEKE~|X-ODrk#nGhXsJ2TC>+`Xu6g(un z%3Pmz-ua^0i|2arye#-|jX8ra_x<^j82FLt62CKn!>J5$=JVaj$;Ch7rd#ux8RRa{FM|7Gmh@2aQv>d2!AA;S zHDcsScPLw5GTrA?Qv3q3P7}JkSyGQr0}+P2SC^u^X(mbPjlskoVViD|Yyl&UN+~7w z`Wq8Ll@t1mxPamc4d5~#yacvW>rhBNY8qG;Z1y)p>6dc$6?fs2^LXrUC6g5n+i(wvCgfpPrBiw0Jexq zrws9ZPHW?OTFHSkN@e17=xy};i{Bj644Tgl%=oFk@RZ~#TOz9)J)he+WGAx%AoY@^ z$2;t<4)rOv@6Z$o8pFQY=9Riig`ORs;*Nsgoa`qCn^_bpH4#dWF(x ziN!O0&rJue=$K0NVJQ!d$^cH(0;@yMXUi8-{7bC2UobKDPuuqyQ#D0?=IKWtz+`_b z+!Lr36j|=d0>KfC__s;kL!_e+qL8%WaV4#ICxdZPvBktIB>1!3c;^yJ4@#k?0Ir#o(N*ugeIIa4yruKE$xx0_FNX?eGqOEo9fnChZd%FN7eRdDdE?X)u`!8_5iO1IS$mCrH>jR+&j&pswOkCx zy}<#;Z*q9>lomvUWJw6|zmW=ISOi~bz1W#|3r7Q`S>i6s|2{HR;iKl~Ec3Wp@OvXP z{i+4O+@Z`13GR6308VRa3Hw&&IJHg-s&~&hY`R`?Pgp-QT zU)@BuA z*EG=t4>WO3D`b=9x4j21?d#s7XWuc`j|YHv!%bz-g|627*uKwNn%8>r8>uXYP2MbE zZlW)WXBw<9PAbZ;X!WSkYj*RKqMR3{A1WK?))T+d#JCH#vgomga_N&>FGpI7U+fyV zNHy8V9qAs|Zb%jFYi6*-d%G{ta)~T{OSTQ-eS)d8H(fPzv#)1Iwr9K481#p~}p?-Di)o1&K z;f4#BWyh%Mp2pngL^@Z+W&$8bdT>RwZ2Gu>+8LY1(+;n zYgUJjHqMC(7CTIK_w&o=pX?v<(kqOQq3uBDhmj8jjgPrf2i-OsH|n%0GH#Wdxns7X zaR^UVm!jO5#Lxy@%c8wq1B5r3L>zYsb{xmM@%2;s`&Xk{bdml~kCpG@x+;5|rn&c7 zj!#tq3EpS49!K7&_e3Q(V}fXlb*oX%*v0k~*1h}Bq4zsnk=bs_`dOVe^PTbY$+-wo zPuo$<{BjL}+BwfUkSVjF&q&I3^6p1F=5lrS7=@dpNnASWm2$6k+v!-hPAwmtJo-5O zVeaxAafNH3_SY*yOvHDY<@L`iQ(3TjyW$9$>IIF26-eTx7P~V~M|UXYdfZIy)?^Y< zS-Kx;{ox~Se~j;rjFxnLZNJ!rm~-CebYt6(TJJl;X%Z3MZU;-gd1?=iHnIQSSH|<$ z9HR@=aHw%zzIcAkYW;IxFAGDC?u(pS_jO!6U)V@=4P=)N?vRb zP1o?%BD`fDFXxckX3Vd&M178vtN)6l@Bjnxu065ph{0#}&@>EsQXXX~nF(Nf>OEl6 zVwgGSZUqjgHkVfa@Hi(reCyu)nLve=OY*0vchS&%3$0K8sPcTfDL((=Z4pPy0S3sK zPxES~T1k+~^Ax!wr=YwG>$iZiuzY2uG@Xti;>fToi+r9-mDN=;Sp}g@O~eP> z><)F2v!TndDdMLKVAq+f5Z`gy)UMmzEsIop;B0WZwDkTfc-tRJSHL4RFK6(UFL5apI$xk55F+pls?sZz@;cdDG zN`FiPi+Sz0BJjP3Z49_9AUGEY$?FGvCA02Hxzqlm3lR`08f!hz>p_bSuV))V zlGJNpx^nLFXj^W=oqDEL*O~z`A-dX&o%q1M5S0HQ_N)qPK*)?w6EE{v zN*Y{3Z-^i0w=_HWNUcdIuKK0yDcUiPG+Hl(#>+W;5b6>8O037*exGxwF=+;GMHOt! z&`S2`T8QQLDcgz14Me=6+|x?}ImuAE9?B8fIJ|l1peef$A}6e#nTaxBUX0Yp-SM#; zrHBla>Z5FT_v#!Gq#mM@bgI`o?RRFT-65aNM>2U|_O|V&gN;d}{SmDOJ_-p+ikQg0 zT-&*E=^}gZP~8<&bw0noBerYy zH5T*inKm01(uP(^#~rSt+<0vLl$~Zi(f2vQvrlyw@i2xa@dG68#_i@q4V+42r^jr; zGaPK;k1esq>lJ@-?nA0fc9|!&wnX@G!TfJuR4iM)8=3?Q0`@TGFeThMzzz0TaZQMH zr;RZGdq0M`0~Ix+C-ZmTuy&{u<7z%*oiE1vB*E85z0y;P&krwDyXCmh0}g<7goW3K z-k8Z?P|3PGua$b%powXSYoHJ2kf6KI*BBaBF3l_UB@&H>?UL1Hd(pJOyFBK&k8js7 z@g%p95L>YQyl(%UMJ{||*3y|-NF(($2A;zbxkQm1pb5v^SPm!r%VF_s4_c4H1l(7p zE`QfznU`WjalCF1Al9r}(S1{&Ms~jll72i@x`v?}6;dM4GcA zXOEX`H&jX-HzS1-Kc6Ae6qFI-4XG|w@oNRvbC6PemZFJK8XjOp$8wrLZc_OLB;8c9 zEJuyfqx72c?6MbUyF6s#6cXfAWb^DRm@zK6d+*MhLkLg22Tw}5+6|Z9)sI9!_j8_| z4YMF~9It_6%)ygjxjqMPr}fjjZOz{)qzce7yb(M9I#^`L`5<1T*wdWF@%KThP{PR@EE1bf zsfx`YW~~bvi$|z-oyfJGax)g2w*^E0V}vMSoYX;#9*$Jl=<(dBQw&t-laO+kG{|Tm zx9Z+sGa5EqEQe1>NbG7{K0AlnNk(IRX6%vbN#2#@NA`&lE_H+UpvIL)t>R=K;6Z%Z zV$_q0hl=Xb^A4JoOh0>jaUw4L$KoT3kOh8$4)M^AcPHg8|NO?3*RVMro$Lk%C|?5s zW8Q2Snj)z_D89fc;GGi(ugauq7*#VvK7ieBdwczX7? ze9tHzNK=fSu8+I=D8@%?s=nO-F$a-M=tkWvFtQ7>E`j<5&#;}H$Y-Y{DL^-I393-X zVi;{k-bA7EKQgfC)wvi;&R?fKsK4^gwJO+d+Kd-!8gnmp{`n;gB*py63RiLQ|6YNx% zf(=~@A$i4=8_t*xh&LUQtuLwd8E|V)ua>^h88M~RwG%8EotLOM10U{@*Fq48w$R@d zn4_9$q!Py~FO;{FGY@98`f^`*E-1x|72V_Gk;7EJn*2xxc3y@Cjp&UyQgd zOVmS>S-|3JQ%AF$kee$?QEI2cpT-E1eRER*4W6QggTI-@Lh}Yk5aoD10Udtz{L1sJ z9d|l?uY_NmQ($!+C0zMzEMuPKAj(UFo`5u5`51){8oZB$LmJ7|RoNrqWh76;YbZ!3NO)_C@xy{V?rvgCk_sKehZpQs6EYwi@yYH{ zLM}vj5pqeYt%EV6C}7eSy?uHvC(z+Ux@n&z z`}NTJtsvx-#BXWi5o2RswaqwxDV|nBNw6g8`9&%VB`XZ$moN>AeCd1be2d_Po1aSQ zbZyUw`IMIWT#YD6eAE#%{6J$;u+ADX*%K}j6^GUP9uO*3A~J$F zM(Bfm^MU*8jcM|48$84(jc+Fp#rKiiC^1`bgsA`lUK-uJAB#L8ffMQZZR5=7uERzg zh=PV<=DX~+n#;sUGCWcH7?^zYAY7F+iB;PoN>{T~DAV zkUWs0c3{s@_Ev8`yLp!#a-F_#7P~4?lv~tL2z9EJI*h#O>kcrj#N|zEx5)$@97c4a zkb$_gnwH-6a-^WkyE|_Byg+7vT|Gto{1eI<27ZfU9u8@rSahpztG*QT?uK(>+F-d( zR;mnAQN6l*l*ab-vKoE9=EI;uoF>{o4f&@zg`e18jtMwUy0N<2~<{3v!;5Rn)XoD4nLW=j;!0)Pk_cn1mv4 zW4kG27t>2-K{n6Ji|pOqdx6Nm?n@8YzdV>I%)^k|lpb>HZ_If)h_M{C zh?x^|0T*0`KENer(tebSmKHQC?A^MP3=z)SBg=0mwY=~6aG--(&#DTuo@c~xyz0_@GjVO9>&A8n$^ORQB_O#IWc55 z@wT%a4^b^n5VpQn7c2jMQ1v4~hRE&GOJ@OFnlvZ zGJCd=Xbj4bqdRpRJpoB@HwE=MP#s#}`-OG5Lk#v5?-am#lm&gS#{i@SwDoAa-E^}E zcu7$ajr5Ib6&Ur(*LS)i)F4*%2{x*I>5LWetLrtQqmK?VPCpLwj#~xXckDaO>c*Y>$j&I$)7qaNATpht@wDGp z%sAYV>3dbpjxFzWTpwg6Ul_l>;_xa%zmcJ*3GQ$lr~w4apGum>8$nC--0X|k6fLp8 zv=R8@ULKSR;kgtQXyFgJC`@f{W+!`$9JTBadf?35k&2qKSOn@J?6Cwk>N5bzoDG!4 z|9Y8;`f15TVerbL_EAV#MA5g3{X=*1LY9kQeQ5!TGAP4VxeQpajU93o-Jyw~ z5DmZ{uJsu%hp+$*{AiW$l zH1|2fFFtt=a^3c-2+a0tP=(`*?)yw{+r7PeNx9@kawa7AnL~K18$8ePw42#s^)#vu zOXgKf4)e9a-9Wl;Y1u#@G4c2A&cKsRR7W=_$BI{wKSwB63iLga}i)vFg=bQi#9SQ6S2uTbr~Sr4W7> zlr(RblVX{#dL@|0-C0A4o`?&~necd?paK_jNSH~xDOp27-W?*>nx=^MTjUmqByscxM9Y&1M(lo9+X182;rB;FMy<#Y5997vV0o zU53a;+-qx@gkJ%oVlkNn(zBIlpRXnn))&r-i zo({-GLml-{Phd8{2c?WS0x-lE&JAlzpAO=JfV~|!=UM>DkY?D@uzO;MnJ?hX9_H+053u*$rpqQpbc~o>Uh?dlkWCpM|k*ke6D^9@I4-Bmp&x3zBE5GI;uON zd>q{sb^aJOQ=@>S?VpVbe^vgRsh&b)FOZ-mSwqOvHV+0qX9Id!B40mO#v50ALQb<80YXJ2_J$bd=}pMQdsee0)h30*Blq1AME1K#BLU|f>*r^3W;6DIB9Pw9ytAfjR%=q||v_^YATvIM|PWU5Nl-(Iux zTH%c2z_|*VnFoAf1$q1glQalHyj|)o+EYwa(VT0Osnaw7gIH2`K|EmDkb2% z?*PBgU|a+lDfI{zp{_7M?!d>g%?iU3k>`6LZt5=AJ5XP)MNCrRyLxqQMt~D&vmu`cgwD*2ldN?I<$GA2!sI=Jj3bj;CZ1&s_ zjTR!kO`N8fd1bD3J`)VX^UcBdddoOj>HNGqWRFt}+aYN);vukxCrJ@$RA|Q~q?(<^ ztJR@$QR*77S_IzJI49MN5ns$G$sf*Wkui#bgpH!UGB04}{5h%O3H2U_NJidpXUp)b zBdKfETY)rF{P0wh>fMg*amHDAa5=)6Ed!S=g8Q+t17TBgNyCf)*X5Pvr37YwcH^K* zf=bXD@yvh%^lCWrrU~rq4-HlW3!MPMqpvGsBb-iHYKXhaSjUG%1oV|nb~g7Jzyco=+ETFrrMro=a{8kml%AQ zw`A5Ka$Z_VbE#vW!#9H@O*~RA)&%C~=J@lF$q&$aL6kEMkV9pRHjeG~B<+Rnd;j1k zmyq+@+bR{!=O#_raFySz2@|e_LsFMiACw5Gtlg<06+bu5h9)Ys=k#l+Tpq_JiE$^b zC8KWM!N0QjDt(__yT5=$>WBD7adQ&=0DMxeYfGuRyXFH!@)e8D2)2lfaSP3~ztWa^ z9|GH>f&YYWqX2EeDW;()er;^=1k1VqJv%2|h`{$=+0L2hpAVPBhie)}pFTyNeJmR( zezfW?*WBH{%D{$epebf1*3ELGRKhRmtao@B9V z*JMcnKwdDpV|Tv_u_GrGKhJ1;0hYsVg7SDPh(R~-O=js<(^i~kqJDh9ZsVDa`Ur`K zs7V#m$|D;s$nOF`a~9RPtG8dxvk}+Fbp@_)gg>rac-`gxss!R*KHBlnK|#qbEmr*8 z<-{GPC6AA42F58lfG5!$U{sI+l$dFwcLyU=TypVi|NQe)mv<*#$>SS~JK$|ZM4khWLieUcs8KrpDT&}ul4KCf%+epr0dII>;x%u;Om!-R0#aw&eHGckq9j21J zQ*uM#S_en_KuAxyYv|4`!V1oBZ~qDB;=v@{zm^qT9~O<4jZ>h1pCTJ2`Ha)6M7{D$ zfg^qGr*wUhTe1IaUpl7(;gsklusez`P;WfaL+WtB%rW;3AjF0RMd;!tNue{|1rW^L z!xW46Nx3>^;mj%135IBL5}W9xl*7EIc02@guh~&a_;(L@j)(zy7_$0L3|&ums|L%f zmRGoa*}ic!I^2DHc&PCjn()?A?8!8^ayZKL(P-j@iz-P0Ny1~*K z#Ly9Ol(~}LRhiUb;=RcxmkC#r;0O%0m;h&m8(`s6260_T_%JkMU#%{H9vWt|_Te3S zpaQD=B@m?+x*H5-=_H?ty$P2z8fL9qU*Y!lPb28KhKH}go^98BpRVryfMYU%aOKGn z<`epkO+&b|Om37xvK1PHK~ms$h!x=ZeU5^QS&E}hMN8c2DKQ(GKY8X&8iTp1@B z$h91JQm0;B!g)7Qyj`0`i(#lp68PZhOQ2dwtrz4-qs0AgJV}~KG8^%Q>fq&9wwVH= zSzYDulK4e=`T=nMrbzh>m_+EdoN7C1wuA)1Z)hc-18i`6iYYz%)$NY+?OnDrt(Ght z`Zk{6;w`u*$#1PyG0L4%&__b8k=)`Rmsho*R1RF)63x79d&R+YE;sXRf;E~WFLA-y zU@DvDD~Nae!6j_seG0_eOvSkJ&}QN-gCP3*)rK1Lt%EpXCP1>^59Ubmj9t*L0AOPg zsVP_rl744z@+4Wp1;n_efs!}G6{35O$4mkC*mG*GWJiBpj3-70Lk-=dM3;}qGes$9 z8a1%#048KSFlsDz$DITG;b1mMXgXD74Q?BIw=|fw!+E!+6#&9)XEKpS7umHoqMgj_ z`VD6$r(?JVyBFV$A+aDmma3s`raUB)2MYoqI9#2@U>%s4zid5PdG^`@qzFk9OnmOP zn)4bk^#Pn?vUM4zzJqFC$0uKJGd(X`vwtJOXyg%(y1u7ZYg3P!h-Of| z5<+uM`oT_@o68=a!ceRG)F6%pjxG{I#<6+*L++K)Ps*2t1$sC_02#@g*Kv){gDRzqaMJ)a>ST9ZiBu z0rTcnu@clV3la=cnfISP*nhy|YGZhGd5TGoSQL2M4YL3?6dn4E+k5v7SaOTtT1!GY zFJ3-^_Z>G1HS%||k-j-89NPEvnh?@Bg4`j9M zBjah(Jo+67DkO+POS0{bu=Nl^I zc8mZwR%s)ROhLjiQ4o;K>l$p*aO5O5K`aF}$u}lF0pM;%-MP?~2+@;Nw*RlaE02dd z|Nm{NO{mtOaz(YSl`Gc_(@;qc@2C%fPMKDL?f@B4V{URi&h3WFyLilsThjCX_i=ZhTP zeBf);4Z;)A2*>H1xzHmP7^nUDTq65NNt5zkC3$6!>CtYgoX6`$J3tO3gFrx-q##%o zRZuc|=i8PhoELz|#~*dCB{0jlZERepWe^*5@FpvBl-uBL;C=+_17bZsrFG3=0?vx= zA=7kL+da9EqMrSXifNgD*?j(&Q*R^}jKxjLpVI(!s;)oE-Pl$JMWUK+c!wV?*IP9= z-5$sSlVj0K><9n7{2c6DwfXK-*8OVgkU*kf($ToO88PNGpTdbk&)*_eFrt)ZOy*N` zIQx>Ef$Fu0?VBS5=7E`zg>bD;00LO~#pO(pqIbB%WLl=eQR>p9K$$b9cY6v!Vs%X! z$nAES4B@@n`kwFZsAjvMp6IN7$>A3CNvxucV$e zUb|~f(Wb{&wL0C8yYaIF-U}~EdSY6&zTcw5F}rNmP_rWTg%tD>(TE&(Q*esKM?cOp=k49zqoY_U zmv?z&#O!ELV0-HA?d#xPF|-&XjV+b?PlTHi@I=jT&tjdg8-)ENuPOuhX00;%>&JbS zP97Gm)QM4j6Mb@5Wah)UR1`)5kLm=+XK>QM4}GiZ-wT9ynlyyqy3TQYOhr^?*F{S% z45P4AnV^-K1)CW6mv%5b(aug%4nzk$@k5)<0`Z+2TnIJJjG$x{i@?UbDw$Rezky*9 z^AV!gLLkg|2Zyt*RVu8MtZ@;Tn}h15f;rwNof&5#;4vECoKvyFTjFI_)(Vqa|FsSa z3=G`S4-eSfdRJIJvS};-C_=URTD*e=z47F!@zjX@A8@F=5s%o822LQ`r+OMYk}29b zktSlAT+q!m)^T>#})L0C}wI?k?xbMH4G*~-_V=AHHe zi2%2$11y`Bj;nKI90t&Cf-$)fvpfB zucF>}G7_);{9HLm{Y(;SZ72hZJN-*qqF1S8O_ zJjgD4ddqqx*-nlj_=j7rW21M(_^h~(y)Dcr3}BT#29EozStmYFoJb=YVVft2Ul^ff zbt3y4KxtOP!nb66wWsl>gRC%*-Rsm)Qf@FpeB(7(5QG}h3Qz|D@$4m6s(&z+sp3Z{ z%t1q9?5D*V{gqq{N$aTjdiFD(7!JT_xKXoT9L$Mma|uniw{jrJ|Ap1ZT~VQmAKK$< zsu%0e;MESj3>Fw4GZ$$NN|ORvfz2>jpTfbnMR7%;j)WqhE;Z5Y#jOO&@zu+sD6T6x zSyWeOLd?@?bC)i!H4030uJVYIBXO> zKEEGIkC)D;DaZgv6fgDeR3-r9*gGX3R=HvErn@>W7Wl}ii3`<=)WseanJw9V-dm@Y zPyE{EwG9z)G~T@0KhebE*4C1g#yhm=k$+TAKts@DR9I&R>XO!l@v$gNEBFQpw6L0C zU}|{u4EE)rhoAOi&*3N{)1g;;as~cPuk@Xe5go&JbfC?K;d1RjRfYmDt2hv`YNQa+ zGjtP0_;qiHYWuPck;h!3Hh^`eWEG(YWYv$g6ghu-?YQX1+Y|^Xe5NmxmIWA`0Sqhm zmBXK9le<7Of`*Y+qhbfh(7*fPpZOqKcRerL zl6zYW}A@=(EVe&@!#{jgzSS>4LRibvEXgnN<4>CL~8Mcwvl8375B z3;r~*ZuVmO<2foB?;UKp`=c}+oHWN*WcS1L<8arGh>j0= zh2L6KM6ry2*U}^*k_GWVXp~Xo3w)+4J|g@OntM6wQG;SAe-5ji@`6FLwXCnq z_*r~yl*28_?pcVhqL*hZ^kh@wbTB&j;a&EsdVW7f3+v7*oYu|sIx=bO24ikC;*GT+ zzq@88c!FDz?hz?-m!>Sj9`DjaE$%^LsC`vz{e90HwFMAtHpYvWYs@7j{+?E1VUl*EWN_crYy*&BeeFf{K zmBH9?wBLFBW8Qwp6SYyfhgoJj`mw+id=bE{xV=zSsuHsqstfg9+%k0tsX@#1&u=oKB?sm@DtIiZ27~wGMdt@U=@o#(5$?7@MImL6U9dEtg=(68Qs=}5|K#fMK_$n2S_x+l3}VB;;fmg~*atU7 zGSf7eYDLLD-#i$CqaxBLyE2_^AXgPV*lL#EK6feB4qD$P{kR6@%(VUu$r8VA9hCAG zet=wNby)TCrmcl&QYSY#HBxL0`#wfMtLG@Ns#7(&e4Pw7+X^Sa2m6gaZo)0SO9>!H z{8SayJwHZ!et*BwLb)p{1nYR77O~^p72hd<@?YM1H5UqJ)BHF#UV z6vk|G5m&>$evk3}bhpeMat??Es#f136j(=|J8|R5x;IHOy6Q&BI{29#vcdtxrlx)8 zWy6d?4?_^Q)gB384Xr~~34D}Z9^buD`>92Ch>#9tL)I*XI3pG#X^?*eL5SR! zXDGG$p<0-vhwS^U_Y@*)o30`@xQR$MX@H~E0eIX8j@e8S>*x5b$8jOB5G?AP+kjto zMR`Tg1*A1We9Tl&8^GI-yV^_KD~VMpa}jAI6hJ+-&gJiV{xQNH)<#a^H+ETJdKcbPDp2UhBke&; zX{@0&XZket{OZe!$1D_I!Ehjkc(Q6)r1}Q8@c!g-`J`tVnFbW)#`7(hamjgPtwU~h zY#J=tIspa!TlHxY)vGcVc6E-Qf$5xBBo;F|Vh?)=s9u#U*s!g3L|v)^WziTG4u#EfESe*! z#)72ko6@uW7w380oG{# zrajaoV-B{Om4^+&`A{jfVS!I?z;yD3K)$VYE?<~yO+P6KyOLimX+8A)S)P3g{&6%Nj=W9#;lF);0B6`e zm4C}bp7g_TLVU>CE5{BeFPx7Pz6*3hVA|V#8oSJczC5oEeoSXC!+k0{0z2ShI<9q9 z1{x*7_rHhQQx4h9-y>P*993vE>%j`(coS>KlzBU>m+LXl1`-p91SF+r{=Lj({yb~}NQl_^yOh1o*%!dsz%!yjCh{fRY?RvK zDSigkVoeuN2;leYqtn03BN)7_l6wvqzutqjRK-LZ0dpkh9gYii0yUUzOe7a5b&e{bQH)$8$gn%itLv` zV5DDz)+6pa{Vc=N;097^8Q|C{qb3N^jjWDgmYWBlfxLMwVaK=J>eufHzqKcy;ByDp z0hSZ|?cs2%-2V{J3GLVj>tFVQ!L-EL7EB`d0dzN*kZ1OzHYf4G=X>if$n?!@u7eUG z3BrJ>WBQqAPXSCQ3b^TH!IL#Rh*~ylBj`jq`jvwR(FKV66tK5F+!Fe~{qc%#(KMm$ z1W>vvi+V7Vrfm+t>INvrf!TucjHe?^f1M^*kuhv8%%0~G+9l(i# zi_F(K73SdkuoU1J0{c>InMUGf#Df%tDr5YfAfU9;5b2d9wD$KwZR^PbA9L^2NDI9n z?~kXu8Cv zJ;vjc_%3i%A%=jMC(?S!cb(8p@s4!YfXOrNF!gN=R&1E6jEgvlBh9!1=Hbw(nRP_O$O}cG5zn5cxv~p9eCy zhhLB2XluM~{2@PGp=$B32Q-vk3FEyuu;5#r-EIrGkoZ27!he32 z1X~H?i{epCpP~oEU;9*o64Mv3a@%&MM_ivDevlv>1v_bep6&PXvFk0#1Nqb$AeBdo zB3kuE3cfy_FUL$lZrYw@r1U1&>=!UJYD0w`@zw;8!#iPB{zeBZ|3nYmL?wY*+TF>T zUxLX`gI#t=G8YMjxZqSq$Df!;d|b0s*r0{1U$clgGyFqWmy+Bx;PTD@N8`K5Vo)<@UBINLCG8MS| z50B+!BLrOF{Q6wazjMk+Jg+coHC$0Q{rj;d}Z`#8aK7O6Pb@1!B2H{BdAyeYN063)~ A+5i9m literal 0 HcmV?d00001 diff --git a/docs/v0.2.0/index.rst b/docs/v0.2.0/index.rst new file mode 100644 index 00000000000..0c054b057d0 --- /dev/null +++ b/docs/v0.2.0/index.rst @@ -0,0 +1,29 @@ +`Envoy Gateway `_ +============= + +Release |version| + +.. image:: https://img.shields.io/badge/slack-join-orange.svg + :target: https://envoyproxy.slack.com/archives/C03E6NHLESV + :alt: Join the Envoy Slack + +Envoy Gateway is an open source project for managing Envoy Proxy as a +standalone or Kubernetes-based application gateway. + +.. note:: + + This project is under active development. Many, many features are not + complete. We would love for you to :doc:`get involved`. + +.. toctree:: + :maxdepth: 1 + + intro/index + intro/compatibility + user_docs + design_docs + dev_docs + releases + roadmap + about_docs + get_involved diff --git a/docs/v0.2.0/intro/compatibility.rst b/docs/v0.2.0/intro/compatibility.rst new file mode 100644 index 00000000000..4dc06f769e5 --- /dev/null +++ b/docs/v0.2.0/intro/compatibility.rst @@ -0,0 +1,19 @@ +Compatibility Matrix +==================== + +Envoy Gateway relies on the Envoy Proxy and the Gateway API, and runs +within a Kubernetes cluster. Not all versions of each of these products +can function together for Envoy Gateway. Supported version combinations +are listed below; **bold** type indicates the versions of the Envoy Proxy +and the Gateway API actually compiled into each Envoy Gateway release. + ++--------------------------+---------------------+---------------------+----------------------------+ +| Envoy Gateway version | Envoy Proxy version | Gateway API version | Kubernetes minimum version | ++--------------------------+---------------------+---------------------+----------------------------+ +| v0.2.0 | **v1.23-latest** | **v0.5.1** | v1.24 | ++--------------------------+---------------------+---------------------+----------------------------+ + +.. note:: + + This project is under active development. Many, many features are not + complete. We would love for you to :doc:`get involved<../get_involved>`. diff --git a/docs/v0.2.0/intro/index.rst b/docs/v0.2.0/intro/index.rst new file mode 100644 index 00000000000..ef349d9d987 --- /dev/null +++ b/docs/v0.2.0/intro/index.rst @@ -0,0 +1,15 @@ +Introduction +============ + +Envoy Gateway is an open source project for managing Envoy Proxy as a +standalone or Kubernetes-based application gateway. Currently, it uses +Gateway API as its sole configuration language. + +Many things are in the scope of Envoy Gateway. Many things are not. Many +things (like support for non-Kubernetes instances) will be in scope later, +but are not now. + +.. note:: + + This project is under active development. Many, many features are not + complete. We would love for you to :doc:`get involved<../get_involved>`. diff --git a/docs/v0.2.0/releases.rst b/docs/v0.2.0/releases.rst new file mode 100644 index 00000000000..090c6707fd2 --- /dev/null +++ b/docs/v0.2.0/releases.rst @@ -0,0 +1,10 @@ +Releases +======== + +Learn more about Envoy Gateway releases. + +.. toctree:: + :maxdepth: 1 + + releases/README + releases/v0.2 diff --git a/docs/v0.2.0/releases/README.md b/docs/v0.2.0/releases/README.md new file mode 100644 index 00000000000..93d3366efb0 --- /dev/null +++ b/docs/v0.2.0/releases/README.md @@ -0,0 +1,41 @@ +# Release Details + +This document provides details for Envoy Gateway releases. Envoy Gateway follows the Semantic Versioning [v2.0.0 spec][] +for release versioning. Since Envoy Gateway is a new project, minor releases are the only defined releases. Envoy +Gateway maintainers will establish additional release details, e.g. patch releases, at a future date. + +## Stable Releases + +Stable releases of Envoy Gateway include: + +* Minor Releases- A new release branch and corresponding tag are created from the `main` branch. A minor release + is supported for 6 months following the release date. As the project matures, Envoy Gateway maintainers will reassess + the support timeframe. + +Minor releases happen quarterly and follow the schedule below. + +## Release Management + +Minor releases are handled by a designated Envoy Gateway maintainer. This maintainer is considered the Release Manager +for the release. The details for creating a release are outlined in the [release guide][]. The Release Manager is +responsible for coordinating the overall release. This includes identifying issues to be fixed in the release, +communications with the Envoy Gateway community, and the mechanics of the release. + +| Quarter | Release Manager | +|:-------:|:--------------------------------------------------------------:| +| 2022 Q4 | Daneyon Hansen ([danehans](https://github.com/danehans)) | +| 2023 Q1 | TBD | + +## Release Schedule + +In order to align with the Envoy Proxy [release schedule][], Envoy Gateway releases are produced on a fixed schedule +(the 22nd day of each quarter), with an acceptable delay of up to 2 weeks, and a hard deadline of 3 weeks. + +| Version | Expected | Actual | Difference | End of Life | +|:-------:|:-----------:|:-----------:|:----------:|:-----------:| +| 0.2.0 | 2022/10/22 | 2022/10/20 | -2 day | 2023/4/20 | +| 0.3.0 | 2023/01/22 | | | | + +[v2.0.0 spec]: https://semver.org/spec/v2.0.0.html +[release guide]: ../dev/releasing.md +[release schedule]: https://github.com/envoyproxy/envoy/blob/main/RELEASES.md#major-release-schedule diff --git a/docs/v0.2.0/releases/v0.2.md b/docs/v0.2.0/releases/v0.2.md new file mode 100644 index 00000000000..a0dc0e885de --- /dev/null +++ b/docs/v0.2.0/releases/v0.2.md @@ -0,0 +1,50 @@ +--- +title: Announcing Envoy Gateway v0.2 +linktitle: v0.2 +subtitle: Major Update +description: Envoy Gateway v0.2 release announcement. +publishdate: 2022-10-20 +release: v0.2.0 +skip_list: true +aliases: +- /releases/v0.2 +- /releases/v0.2.0 +--- +# Envoy Gateway Release v0.2 + +We are pleased to announce the release of Envoy Gateway v0.2! + +This is the first functional release of Envoy Gateway. We would like to thank the entire Envoy Gateway community for +helping publish the release. + +| [Release Notes][] | [Docs][docs] | [Compatibility Matrix][matrix] | [Download][] | +|-------------------|--------------|--------------------------------|--------------| + +## What's New + +The release adds a ton of features and functionality. Here are some highlights: + +### Kubernetes Support + +Run Envoy Gateway in a Kubernetes cluster. Checkout the [quickstart guide][] to get started with Envoy Gateway in a few +simple steps. + +### Gateway API Support + +Envoy Gateway supports Gateway API resources for running and configuring a managed fleet of Envoy proxies. Envoy Gateway +passes Gateway API core [conformance tests][] and supports GatewayClass, Gateway, HTTPRoute, and TLSRoute resources. See +the [documentation][docs] for additional details on how to use Envoy Gateway for your edge proxy and API gateway needs. + +## Envoy Gateway at EnvoyCon NA + +Envoy Gateway will be at [EnvoyCon NA][] this October in Detroit. Don't miss [our talk][] to learn more about the +release and future direction of the project. + +[Release Notes]: https://github.com/envoyproxy/gateway/blob/main/release-notes/v0.2.0.yaml +[matrix]: https://gateway.envoyproxy.io/intro/compatibility.html +[docs]: https://gateway.envoyproxy.io/index.html +[Download]: https://github.com/envoyproxy/gateway/releases/tag/v0.2.0 +[conformance tests]: https://gateway-api.sigs.k8s.io/concepts/conformance/?h=conformance +[quickstart guide]: https://gateway.envoyproxy.io/user/quickstart.html +[EnvoyCon NA]: https://events.linuxfoundation.org/envoycon-north-america/program/schedule/ +[our talk]: https://sched.co/1AO5S diff --git a/docs/v0.2.0/roadmap.rst b/docs/v0.2.0/roadmap.rst new file mode 100644 index 00000000000..711b6245503 --- /dev/null +++ b/docs/v0.2.0/roadmap.rst @@ -0,0 +1,9 @@ +Roadmap +======= + +Learn about the future direction of Envoy Gateway. + +.. toctree:: + :maxdepth: 2 + + design/roadmap diff --git a/docs/v0.2.0/user/http-redirect.md b/docs/v0.2.0/user/http-redirect.md new file mode 100644 index 00000000000..9762c798895 --- /dev/null +++ b/docs/v0.2.0/user/http-redirect.md @@ -0,0 +1,123 @@ +# HTTP Redirects + +The [HTTPRoute][] resource can issue redirects to clients or rewrite paths sent upstream using filters. Note that +HTTPRoute rules cannot use both filter types at once. Currently, Envoy Gateway only supports __core__ +[HTTPRoute filters][] which consist of `RequestRedirect` and `RequestHeaderModifier` at the time of this writing. To +learn more about HTTP routing, refer to the [Gateway API documentation][]. + +Follow the steps from the [Secure Gateways](secure-gateways.md) to install Envoy Gateway and the example manifest. Do not +proceed until you can curl the example backend from the Quickstart guide using HTTPS. + +## Redirects +Redirects return HTTP 3XX responses to a client, instructing it to retrieve a different resource. A +[`RequestRedirect` filter][req_filter] instructs Gateways to emit a redirect response to requests that match the rule. +For example, to issue a permanent redirect (301) from HTTP to HTTPS, configure `requestRedirect.statusCode=301` and +`requestRedirect.scheme="https"`: + +```shell +cat < GET /get HTTP/1.1 +> Host: headers.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< +... + "headers": { + "Accept": [ + "*/*" + ], + "Add-Header": [ + "something", + "foo" + ], +... +``` + +## Setting Request Headers + +Setting headers is similar to adding headers. If the request does not have the header configured by the filter, then it will be added, but unlike [adding request headers](#adding-request-headers) which will append the value of the header if the request already contains it, setting a header will cause the value to be replaced by the value configured in the filter. + +```shell +cat < GET /get HTTP/1.1 +> Host: headers.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< + "headers": { + "Accept": [ + "*/*" + ], + "Set-Header": [ + "foo" + ], +... +``` + +## Removing Request Headers + +Headers can be removed from a request by simply supplying a list of header names. + +Setting headers is similar to adding headers. If the request does not have the header configured by the filter, then it will be added, but unlike [adding request headers](#adding-request-headers) which will append the value of the header if the request already contains it, setting a header will cause the value to be replaced by the value configured in the filter. + +```shell +cat < GET /get HTTP/1.1 +> Host: headers.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< + + "headers": { + "Accept": [ + "*/*" + ], + "Add-Header": [ + "something" + ], +... +``` + +## Combining Filters + +Headers can be added/set/removed in separate filters on the same HTTPRoute and they will all perform as expected + +```shell +cat < GET /get HTTP/1.1 +> Host: backends.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< +... + "namespace": "default", + "ingress": "", + "service": "", + "pod": "backend-79665566f5-s589f" +... +``` + +## Multiple backendRefs + +If multiple backendRefs are configured, then traffic will be split between the backendRefs equally unless a weight is configured. + +First, create a second instance of the example app from the quickstart: + +```shell +cat < GET /get HTTP/1.1 +> Host: backends.example +> User-Agent: curl/7.81.0 +> Accept: */* +> add-header: something +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< content-type: application/json +< x-content-type-options: nosniff +< content-length: 474 +< x-envoy-upstream-service-time: 0 +< server: envoy +< +... + "namespace": "default", + "ingress": "", + "service": "", + "pod": "backend-75bcd4c969-lsxpz" +... +``` + +## Weighted backendRefs + +If multiple backendRefs are configured and an un-even traffic split between the backends is desired, then the `weight` field can be used to control the weight of requests to each backend. If weight is not configured for a backendRef it is assumed to be `1`. + +The [weight field in a backendRef][backendRefs] controls the distribution of the traffic split. The proportion of requests to a single backendRef is calculated by dividing its `weight` by the sum of all backendRef weights in the HTTPRoute. The weight is not a percentage and the sum of all weights does not need to add up to 100. + +The HTTPRoute below will configure the gateway to send 80% of the traffic to the backend service, and 20% to the backend-2 service. + +```shell +cat < GET /get HTTP/1.1 +> Host: backends.example +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 500 Internal Server Error +< server: envoy +< content-length: 0 +< +``` + +[HTTPRoute]: https://gateway-api.sigs.k8s.io/api-types/httproute/ +[backendRefs]: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.BackendRef diff --git a/docs/user/quickstart.md b/docs/v0.2.0/user/quickstart.md similarity index 100% rename from docs/user/quickstart.md rename to docs/v0.2.0/user/quickstart.md diff --git a/docs/v0.2.0/user/secure-gateways.md b/docs/v0.2.0/user/secure-gateways.md new file mode 100644 index 00000000000..e1be52ef111 --- /dev/null +++ b/docs/v0.2.0/user/secure-gateways.md @@ -0,0 +1,260 @@ +# Secure Gateways + +This guide will help you get started using secure Gateways. The guide uses a self-signed CA, so it should be used for +testing and demonstration purposes only. + +## Prerequisites + +- A Kubernetes cluster with `kubectl` context configured for the cluster. +- OpenSSL to generate TLS assets. + +__Note:__ Envoy Gateway is tested against Kubernetes v1.24. + +## Installation + +Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and the example manifest. +Before proceeding, you should be able to query the example backend using HTTP. + +## TLS Certificates + +Generate the certificates and keys used by the Gateway to terminate client TLS connections. + +Create a root certificate and private key to sign certificates: + +```shell +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=example Inc./CN=example.com' -keyout example.com.key -out example.com.crt +``` + +Create a certificate and a private key for `www.example.com`: + +```shell +openssl req -out www.example.com.csr -newkey rsa:2048 -nodes -keyout www.example.com.key -subj "/CN=www.example.com/O=httpbin organization" +openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in www.example.com.csr -out www.example.com.crt +``` + +Store the cert/key in a Secret: + +```shell +kubectl create secret tls example-cert --key=www.example.com.key --cert=www.example.com.crt +``` + +Update the Gateway from the Quickstart guide to include an HTTPS listener that listens on port `8443` and references the +`example-cert` Secret: + +```shell +kubectl patch gateway eg --type=json --patch '[{ + "op": "add", + "path": "/spec/listeners/-", + "value": { + "name": "https", + "protocol": "HTTPS", + "port": 8443, + "tls": { + "mode": "Terminate", + "certificateRefs": [{ + "kind": "Secret", + "group": "", + "name": "example-cert", + }], + }, + }, +}]' +``` + +Verify the Gateway status: + +```shell +kubectl get gateway/eg -o yaml +``` + +## Testing + +### Clusters without External LoadBalancer Support + +Get the name of the Envoy service created the by the example Gateway: + +```shell +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') +``` + +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8043:8443 & +``` + +Query the example app through Envoy proxy: + +```shell +curl -v -HHost:www.example.com --resolve "www.example.com:8043:127.0.0.1" \ +--cacert example.com.crt https://www.example.com:8043/get +``` + +### Clusters with External LoadBalancer Support + +Get the External IP of the Gateway: + +```shell +export GATEWAY_HOST=$(kubectl get gateway/eg -o jsonpath='{.status.addresses[0].value}') +``` + +Query the example app through the Gateway: + +```shell +curl -v -HHost:www.example.com --resolve "www.example.com:8443:${GATEWAY_HOST}" \ +--cacert example.com.crt https://www.example.com:8443/get +``` + +## Multiple HTTPS Listeners + +Create a TLS cert/key for the additional HTTPS listener: + +```shell +openssl req -out foo.example.com.csr -newkey rsa:2048 -nodes -keyout foo.example.com.key -subj "/CN=foo.example.com/O=httpbin organization" +openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in foo.example.com.csr -out foo.example.com.crt +``` + +Store the cert/key in a Secret: + +```shell +kubectl create secret tls foo-cert --key=foo.example.com.key --cert=foo.example.com.crt +``` + +Create another HTTPS listener on the example Gateway: + +```shell +kubectl patch gateway eg --type=json --patch '[{ + "op": "add", + "path": "/spec/listeners/-", + "value": { + "name": "https-foo", + "protocol": "HTTPS", + "port": 8443, + "hostname": "foo.example.com", + "tls": { + "mode": "Terminate", + "certificateRefs": [{ + "kind": "Secret", + "group": "", + "name": "foo-cert", + }], + }, + }, +}]' +``` + +Update the HTTPRoute to route traffic for hostname `foo.example.com` to the example backend service: + +```shell +kubectl patch httproute backend --type=json --patch '[{ + "op": "add", + "path": "/spec/hostnames/-", + "value": "foo.example.com", +}]' +``` + +Verify the Gateway status: + +```shell +kubectl get gateway/eg -o yaml +``` + +Follow the steps in the [Testing section](#testing) to test connectivity to the backend app through both Gateway +listeners. Replace `www.example.com` with `foo.example.com` to test the new HTTPS listener. + +## Cross Namespace Certificate References + +A Gateway can be configured to reference a certificate in a different namespace. This is allowed by a [ReferenceGrant][] +created in the target namespace. Without the ReferenceGrant, a cross-namespace reference is invalid. + +Before proceeding, ensure you can query the HTTPS backend service from the [Testing section](#testing). + +To demonstrate cross namespace certificate references, create a ReferenceGrant that allows Gateways from the "default" +namespace to reference Secrets in the "envoy-gateway-system" namespace: + +```console +$ cat < Updated Release Version: $(TAG)\033[0m" + $(eval LAST_VERSION := $(shell cat VERSION)) + cat docs/index.html | sed "s;$(LAST_VERSION);$(TAG);g" > $(OUTPUT_DIR)/index.html + mv $(OUTPUT_DIR)/index.html docs/index.html + echo $(TAG) > VERSION + +.PHONY: docs-release-gen +docs-release-gen: + @echo "\033[36m===========> Added Release Doc: docs/$(TAG)\033[0m" + cp -r docs/latest docs/$(TAG) + @for DOC in $(shell ls docs/latest/user); do \ + cp docs/$(TAG)/user/$$DOC $(OUTPUT_DIR)/$$DOC ; \ + cat $(OUTPUT_DIR)/$$DOC | sed "s;latest;$(TAG);g" > $(OUTPUT_DIR)/$(TAG)-$$DOC ; \ + mv $(OUTPUT_DIR)/$(TAG)-$$DOC docs/$(TAG)/user/$$DOC ; \ + echo "\033[36m===========> Updated: docs/$(TAG)/user/$$DOC\033[0m" ; \ + done diff --git a/tools/make/kube.mk b/tools/make/kube.mk index c78929bf09a..1c1b58b8c9f 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -115,10 +115,3 @@ generate-manifests: $(tools/kustomize) ## Generate Kubernetes release manifests. generate-artifacts: generate-manifests ## Generate release artifacts. cp -r $(ROOT_DIR)/release-notes/$(TAG).yaml $(OUTPUT_DIR)/release-notes.yaml @echo "\033[36m===========> Added: $(OUTPUT_DIR)/release-notes.yaml\033[0m" - -.PHONY: update-quickstart -update-quickstart: ## Update quickstart doc image tags to a specific version. - cp -r docs/user/quickstart.md $(OUTPUT_DIR)/quickstart.md - cat $(OUTPUT_DIR)/quickstart.md | sed "s;latest;$(TAG);g" > $(OUTPUT_DIR)/quickstart-$(TAG).md - mv $(OUTPUT_DIR)/quickstart-$(TAG).md docs/user/quickstart.md - @echo "\033[36m===========> Updated: docs/user/quickstart.md\033[0m" From bb73c6822e7b42c3af0050ac03c496865fe48443 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Fri, 4 Nov 2022 00:38:05 +0800 Subject: [PATCH 088/113] feat: support user-facing version (#609) --- docs/latest/conf.py | 1 + internal/cmd/root.go | 5 ++-- internal/cmd/version/version.go | 42 +++++++++++++++++++++++++++++++++ internal/cmd/versions.go | 36 ++++------------------------ tools/make/golang.mk | 9 ++++++- tools/make/tools.mk | 1 - tools/src/goversion/go.mod | 13 ---------- tools/src/goversion/go.sum | 16 ------------- tools/src/goversion/pin.go | 11 --------- 9 files changed, 59 insertions(+), 75 deletions(-) create mode 100644 internal/cmd/version/version.go delete mode 100644 tools/src/goversion/go.mod delete mode 100644 tools/src/goversion/go.sum delete mode 100644 tools/src/goversion/pin.go diff --git a/docs/latest/conf.py b/docs/latest/conf.py index c90c4a49fef..e5d4e3e423b 100644 --- a/docs/latest/conf.py +++ b/docs/latest/conf.py @@ -31,6 +31,7 @@ } variables_to_export = [ + "version", "envoyVersion", "gatewayAPIVersion", ] diff --git a/internal/cmd/root.go b/internal/cmd/root.go index ae4c96eb8e4..7b57c72d7a4 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -14,13 +14,14 @@ import ( func GetRootCommand() *cobra.Command { cmd := &cobra.Command{ Use: "envoy-gateway", - Short: "Manages Envoy Proxy as a standalone or Kubernetes-based application gateway", + Short: "Envoy Gateway", + Long: "Manages Envoy Proxy as a standalone or Kubernetes-based application gateway", } cmd.AddCommand(getServerCommand()) + cmd.AddCommand(getVersionsCommand()) cmd.AddCommand(getxDSTestCommand()) cmd.AddCommand(getCertGenCommand()) - cmd.AddCommand(getVersionsCommand()) return cmd } diff --git a/internal/cmd/version/version.go b/internal/cmd/version/version.go new file mode 100644 index 00000000000..7315b27db0e --- /dev/null +++ b/internal/cmd/version/version.go @@ -0,0 +1,42 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package version + +import ( + "fmt" + "runtime/debug" + "strings" + + "github.com/envoyproxy/gateway/internal/ir" +) + +var ( + EnvoyGatewayVersion string + GatewayAPIVersion string + EnvoyVersion = strings.Split(ir.DefaultProxyImage, ":")[1] + GitCommitID string +) + +func init() { + bi, ok := debug.ReadBuildInfo() + if ok { + for _, dep := range bi.Deps { + if dep.Path == "sigs.k8s.io/gateway-api" { + GatewayAPIVersion = dep.Version + } + } + } +} + +// Print shows the versions of the Envoy Gateway. +func Print() error { + fmt.Printf("ENVOY_GATEWAY_VERSION: %s\n", EnvoyGatewayVersion) + fmt.Printf("ENVOY_VERSION: %s\n", EnvoyVersion) + fmt.Printf("GATEWAYAPI_VERSION: %s\n", GatewayAPIVersion) + fmt.Printf("GIT_COMMIT_ID: %s\n", GitCommitID) + + return nil +} diff --git a/internal/cmd/versions.go b/internal/cmd/versions.go index b95cbade35c..59a96bbe51c 100644 --- a/internal/cmd/versions.go +++ b/internal/cmd/versions.go @@ -7,10 +7,8 @@ package cmd import ( "fmt" - "runtime/debug" - "strings" - "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/cmd/version" "github.com/spf13/cobra" ) @@ -36,36 +34,12 @@ func getVersionsCommand() *cobra.Command { // versions shows the versions of the Envoy Gateway. func versions(envOutput bool) error { - envoyVersion := strings.Split(ir.DefaultProxyImage, ":")[1] - if envOutput { - fmt.Printf("ENVOY_VERSION=\"%s\"\n", envoyVersion) + fmt.Printf("ENVOY_VERSION=\"%s\"\n", version.EnvoyVersion) + fmt.Printf("GATEWAYAPI_VERSION=\"%s\"\n", version.GatewayAPIVersion) + fmt.Printf("ENVOY_GATEWAY_VERSION=\"%s\"\n", version.EnvoyGatewayVersion) } else { - fmt.Printf("Envoy: %s\n", envoyVersion) - } - - bi, ok := debug.ReadBuildInfo() - if !ok { - return fmt.Errorf("could not read build info") - } - - foundGatewayAPI := false - - for _, dep := range bi.Deps { - if dep.Path == "sigs.k8s.io/gateway-api" { - if envOutput { - fmt.Printf("GATEWAYAPI_VERSION=\"%s\"\n", dep.Version) - } else { - fmt.Printf("Gateway API: %s\n", dep.Version) - } - - foundGatewayAPI = true - break - } - } - - if !foundGatewayAPI { - return fmt.Errorf("could not find Gateway API version") + return version.Print() } return nil diff --git a/tools/make/golang.mk b/tools/make/golang.mk index d6fc6a97d05..9bb4a558466 100644 --- a/tools/make/golang.mk +++ b/tools/make/golang.mk @@ -2,6 +2,13 @@ # # All make targets related to golang are defined in this file. +VERSION_PACKAGE := github.com/envoyproxy/gateway/internal/cmd/version + +GO_LDFLAGS += -X $(VERSION_PACKAGE).EnvoyGatewayVersion=$(shell cat VERSION) \ + -X $(VERSION_PACKAGE).GitCommitID=$(GIT_COMMIT) + +GIT_COMMIT:=$(shell git rev-parse HEAD) + GOPATH := $(shell go env GOPATH) ifeq ($(origin GOBIN), undefined) GOBIN := $(GOPATH)/bin @@ -20,7 +27,7 @@ go.build.%: $(eval OS := $(word 1,$(subst _, ,$(PLATFORM)))) $(eval ARCH := $(word 2,$(subst _, ,$(PLATFORM)))) @$(call log, "Building binary $(COMMAND) with commit $(REV) for $(OS) $(ARCH)") - CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o $(OUTPUT_DIR)/$(OS)/$(ARCH)/$(COMMAND) $(ROOT_PACKAGE)/cmd/$(COMMAND) + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o $(OUTPUT_DIR)/$(OS)/$(ARCH)/$(COMMAND) -ldflags "$(GO_LDFLAGS)" $(ROOT_PACKAGE)/cmd/$(COMMAND) # Build the envoy-gateway binaries in the hosted platforms. .PHONY: go.build diff --git a/tools/make/tools.mk b/tools/make/tools.mk index 6495120ea43..d945caf79e8 100644 --- a/tools/make/tools.mk +++ b/tools/make/tools.mk @@ -14,7 +14,6 @@ $(tools.bindir)/%: $(tools.srcdir)/%.sh # tools/controller-gen = $(tools.bindir)/controller-gen tools/golangci-lint = $(tools.bindir)/golangci-lint -tools/goversion = $(tools.bindir)/goversion tools/kustomize = $(tools.bindir)/kustomize tools/kind = $(tools.bindir)/kind tools/setup-envtest = $(tools.bindir)/setup-envtest diff --git a/tools/src/goversion/go.mod b/tools/src/goversion/go.mod deleted file mode 100644 index 68a8691075e..00000000000 --- a/tools/src/goversion/go.mod +++ /dev/null @@ -1,13 +0,0 @@ -module local - -go 1.19 - -require github.com/emissary-ingress/goversion v0.0.0-20220825220041-6870ba273e76 - -require ( - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/spf13/cobra v1.5.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/mod v0.5.1 // indirect - golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 // indirect -) diff --git a/tools/src/goversion/go.sum b/tools/src/goversion/go.sum deleted file mode 100644 index b1054153db3..00000000000 --- a/tools/src/goversion/go.sum +++ /dev/null @@ -1,16 +0,0 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/emissary-ingress/goversion v0.0.0-20220825220041-6870ba273e76 h1:r5LJ+pvXBngyIBaL9u10pXNfd332eD5JfjR5duY8TEk= -github.com/emissary-ingress/goversion v0.0.0-20220825220041-6870ba273e76/go.mod h1:O3FGqM30w7Xc2n6cZg7mdZa6o+u1AsxEfcdnqKNFDic= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/src/goversion/pin.go b/tools/src/goversion/pin.go deleted file mode 100644 index 83faf7ad75d..00000000000 --- a/tools/src/goversion/pin.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -//go:build pin -// +build pin - -package ignore - -import "github.com/emissary-ingress/goversion" From c1472189f98818b4704392d81cddbefe1fc51ff3 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 3 Nov 2022 11:10:56 -0700 Subject: [PATCH 089/113] Fixes README Broken Links (#693) Signed-off-by: danehans --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6a4e2bd8ff1..6f6e80f993e 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Kubernetes-based application gateway. * [Blog][blog] introducing Envoy Gateway. * [Goals](GOALS.md) -* [Quickstart](./docs/user/quickstart.md) to use Envoy Gateway in a few simple steps. -* [Roadmap](./docs/design/roadmap.md) +* [Quickstart](./docs/latest/user/quickstart.md) to use Envoy Gateway in a few simple steps. +* [Roadmap](./docs/latest/design/roadmap.md) ## Contact @@ -20,9 +20,9 @@ Kubernetes-based application gateway. ## Contributing -* [Code of conduct](./docs/dev/CODE_OF_CONDUCT.md) -* [Contributing guide](./docs/dev/CONTRIBUTING.md) -* [Developer guide](docs/dev/README.md) +* [Code of conduct](./docs/latest/dev/CODE_OF_CONDUCT.md) +* [Contributing guide](./docs/latest/dev/CONTRIBUTING.md) +* [Developer guide](docs/latest/dev/README.md) ## Community Meeting From 80c78a59197218c9caa34f9c4ed0995a067a3bf2 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 3 Nov 2022 17:42:35 -0600 Subject: [PATCH 090/113] run conformance tests on three Kubernetes versions (#681) * run conformance tests on three Kubernetes versions Closes #493. Signed-off-by: Steve Kriss * serialize conformance runs on single runner Signed-off-by: Steve Kriss --- .github/workflows/build_and_test.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 03d062b2ea1..d0976ede653 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -58,8 +58,23 @@ jobs: run: make build-multiarch PLATFORMS="linux_amd64 linux_arm64" # conformance - - name: Run Conformance Tests - run: CONFORMANCE_UNIQUE_PORTS=false make conformance + - name: Run Conformance Tests (v1.24.0) + env: + KIND_NODE_TAG: v1.24.0 + CONFORMANCE_UNIQUE_PORTS: false + run: make conformance + + - name: Run Conformance Tests (v1.23.6) + env: + KIND_NODE_TAG: v1.23.6 + CONFORMANCE_UNIQUE_PORTS: false + run: make conformance + + - name: Run Conformance Tests (v1.22.9) + env: + KIND_NODE_TAG: v1.22.9 + CONFORMANCE_UNIQUE_PORTS: false + run: make conformance # build and push image - name: Login to DockerHub From 976366f606f1369202272bd3d2de71ecd99edaa5 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Fri, 4 Nov 2022 08:25:22 -0700 Subject: [PATCH 091/113] Updates User Docs to Use Echoserver (#694) Signed-off-by: danehans Signed-off-by: danehans --- docs/latest/user/http-redirect.md | 12 ++++++------ docs/latest/user/secure-gateways.md | 4 ++-- docs/v0.2.0/user/http-redirect.md | 12 ++++++------ docs/v0.2.0/user/secure-gateways.md | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/latest/user/http-redirect.md b/docs/latest/user/http-redirect.md index 9762c798895..c11b346b592 100644 --- a/docs/latest/user/http-redirect.md +++ b/docs/latest/user/http-redirect.md @@ -5,8 +5,8 @@ HTTPRoute rules cannot use both filter types at once. Currently, Envoy Gateway o [HTTPRoute filters][] which consist of `RequestRedirect` and `RequestHeaderModifier` at the time of this writing. To learn more about HTTP routing, refer to the [Gateway API documentation][]. -Follow the steps from the [Secure Gateways](secure-gateways.md) to install Envoy Gateway and the example manifest. Do not -proceed until you can curl the example backend from the Quickstart guide using HTTPS. +Follow the steps from the [Secure Gateways](secure-gateways.md) to install Envoy Gateway and the example manifest. Do +not proceed until you can curl the example backend from the Quickstart guide using HTTPS. ## Redirects Redirects return HTTP 3XX responses to a client, instructing it to retrieve a different resource. A @@ -34,8 +34,8 @@ spec: hostname: www.example.com port: 8443 backendRefs: - - name: httpbin - port: 80 + - name: backend + port: 3000 EOF ``` @@ -95,8 +95,8 @@ spec: replaceFullPath: /status/200 statusCode: 302 backendRefs: - - name: httpbin - port: 80 + - name: backend + port: 3000 EOF ``` diff --git a/docs/latest/user/secure-gateways.md b/docs/latest/user/secure-gateways.md index e1be52ef111..9fec46c59bf 100644 --- a/docs/latest/user/secure-gateways.md +++ b/docs/latest/user/secure-gateways.md @@ -28,7 +28,7 @@ openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=example In Create a certificate and a private key for `www.example.com`: ```shell -openssl req -out www.example.com.csr -newkey rsa:2048 -nodes -keyout www.example.com.key -subj "/CN=www.example.com/O=httpbin organization" +openssl req -out www.example.com.csr -newkey rsa:2048 -nodes -keyout www.example.com.key -subj "/CN=www.example.com/O=example organization" openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in www.example.com.csr -out www.example.com.crt ``` @@ -110,7 +110,7 @@ curl -v -HHost:www.example.com --resolve "www.example.com:8443:${GATEWAY_HOST}" Create a TLS cert/key for the additional HTTPS listener: ```shell -openssl req -out foo.example.com.csr -newkey rsa:2048 -nodes -keyout foo.example.com.key -subj "/CN=foo.example.com/O=httpbin organization" +openssl req -out foo.example.com.csr -newkey rsa:2048 -nodes -keyout foo.example.com.key -subj "/CN=foo.example.com/O=example organization" openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in foo.example.com.csr -out foo.example.com.crt ``` diff --git a/docs/v0.2.0/user/http-redirect.md b/docs/v0.2.0/user/http-redirect.md index 9762c798895..c11b346b592 100644 --- a/docs/v0.2.0/user/http-redirect.md +++ b/docs/v0.2.0/user/http-redirect.md @@ -5,8 +5,8 @@ HTTPRoute rules cannot use both filter types at once. Currently, Envoy Gateway o [HTTPRoute filters][] which consist of `RequestRedirect` and `RequestHeaderModifier` at the time of this writing. To learn more about HTTP routing, refer to the [Gateway API documentation][]. -Follow the steps from the [Secure Gateways](secure-gateways.md) to install Envoy Gateway and the example manifest. Do not -proceed until you can curl the example backend from the Quickstart guide using HTTPS. +Follow the steps from the [Secure Gateways](secure-gateways.md) to install Envoy Gateway and the example manifest. Do +not proceed until you can curl the example backend from the Quickstart guide using HTTPS. ## Redirects Redirects return HTTP 3XX responses to a client, instructing it to retrieve a different resource. A @@ -34,8 +34,8 @@ spec: hostname: www.example.com port: 8443 backendRefs: - - name: httpbin - port: 80 + - name: backend + port: 3000 EOF ``` @@ -95,8 +95,8 @@ spec: replaceFullPath: /status/200 statusCode: 302 backendRefs: - - name: httpbin - port: 80 + - name: backend + port: 3000 EOF ``` diff --git a/docs/v0.2.0/user/secure-gateways.md b/docs/v0.2.0/user/secure-gateways.md index e1be52ef111..9fec46c59bf 100644 --- a/docs/v0.2.0/user/secure-gateways.md +++ b/docs/v0.2.0/user/secure-gateways.md @@ -28,7 +28,7 @@ openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=example In Create a certificate and a private key for `www.example.com`: ```shell -openssl req -out www.example.com.csr -newkey rsa:2048 -nodes -keyout www.example.com.key -subj "/CN=www.example.com/O=httpbin organization" +openssl req -out www.example.com.csr -newkey rsa:2048 -nodes -keyout www.example.com.key -subj "/CN=www.example.com/O=example organization" openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in www.example.com.csr -out www.example.com.crt ``` @@ -110,7 +110,7 @@ curl -v -HHost:www.example.com --resolve "www.example.com:8443:${GATEWAY_HOST}" Create a TLS cert/key for the additional HTTPS listener: ```shell -openssl req -out foo.example.com.csr -newkey rsa:2048 -nodes -keyout foo.example.com.key -subj "/CN=foo.example.com/O=httpbin organization" +openssl req -out foo.example.com.csr -newkey rsa:2048 -nodes -keyout foo.example.com.key -subj "/CN=foo.example.com/O=example organization" openssl x509 -req -days 365 -CA example.com.crt -CAkey example.com.key -set_serial 0 -in foo.example.com.csr -out foo.example.com.crt ``` From 54d44d57df53bd9ea9dec77f08a51fdbb0b28ed9 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Sat, 5 Nov 2022 01:50:38 -0700 Subject: [PATCH 092/113] Cleans-Up Docs (#640) Signed-off-by: danehans --- docs/latest/about_docs.rst | 2 +- docs/latest/get_involved.rst | 2 +- docs/latest/index.rst | 21 ++++++++++++--------- docs/latest/intro/index.rst | 15 --------------- docs/latest/user_docs.rst | 2 +- docs/v0.2.0/about_docs.rst | 2 +- docs/v0.2.0/get_involved.rst | 2 +- docs/v0.2.0/index.rst | 21 ++++++++++++--------- docs/v0.2.0/intro/index.rst | 15 --------------- docs/v0.2.0/user_docs.rst | 2 +- 10 files changed, 30 insertions(+), 54 deletions(-) delete mode 100644 docs/latest/intro/index.rst delete mode 100644 docs/v0.2.0/intro/index.rst diff --git a/docs/latest/about_docs.rst b/docs/latest/about_docs.rst index 64a12d791d7..ecaa28247b9 100644 --- a/docs/latest/about_docs.rst +++ b/docs/latest/about_docs.rst @@ -1,4 +1,4 @@ -About the documentation +About the Documentation ======================= Learn how to contribute to Envoy Gateway documentation. diff --git a/docs/latest/get_involved.rst b/docs/latest/get_involved.rst index cd4a70b07ce..f17febd5651 100644 --- a/docs/latest/get_involved.rst +++ b/docs/latest/get_involved.rst @@ -1,4 +1,4 @@ -Getting involved +Getting Involved ================ We welcome contributions from the community. Please carefully review the diff --git a/docs/latest/index.rst b/docs/latest/index.rst index 0c054b057d0..39f345fa125 100644 --- a/docs/latest/index.rst +++ b/docs/latest/index.rst @@ -1,24 +1,19 @@ `Envoy Gateway `_ ============= -Release |version| +Release: |version| .. image:: https://img.shields.io/badge/slack-join-orange.svg :target: https://envoyproxy.slack.com/archives/C03E6NHLESV :alt: Join the Envoy Slack -Envoy Gateway is an open source project for managing Envoy Proxy as a -standalone or Kubernetes-based application gateway. - -.. note:: - - This project is under active development. Many, many features are not - complete. We would love for you to :doc:`get involved`. +Envoy Gateway is an open source project for managing `Envoy Proxy`_ as a standalone or Kubernetes-based application +gateway. `Gateway API`_ resources are used to dynamically provision and configure the managed Envoy Proxies. Whether +you are interested in using or contributing to Envoy Gateway, the following resources will help you get started: .. toctree:: :maxdepth: 1 - intro/index intro/compatibility user_docs design_docs @@ -27,3 +22,11 @@ standalone or Kubernetes-based application gateway. roadmap about_docs get_involved + +.. note:: + + This project is under active development. Many, many features are not + complete. We would love for you to :doc:`get involved`. + +.. _Envoy Proxy: https://www.envoyproxy.io/ +.. _Gateway API: https://gateway-api.sigs.k8s.io/ diff --git a/docs/latest/intro/index.rst b/docs/latest/intro/index.rst deleted file mode 100644 index ef349d9d987..00000000000 --- a/docs/latest/intro/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -Introduction -============ - -Envoy Gateway is an open source project for managing Envoy Proxy as a -standalone or Kubernetes-based application gateway. Currently, it uses -Gateway API as its sole configuration language. - -Many things are in the scope of Envoy Gateway. Many things are not. Many -things (like support for non-Kubernetes instances) will be in scope later, -but are not now. - -.. note:: - - This project is under active development. Many, many features are not - complete. We would love for you to :doc:`get involved<../get_involved>`. diff --git a/docs/latest/user_docs.rst b/docs/latest/user_docs.rst index 2f4f77719cf..5a2f83e312e 100644 --- a/docs/latest/user_docs.rst +++ b/docs/latest/user_docs.rst @@ -4,7 +4,7 @@ User Guides Learn how to deploy, use, and operate Envoy Gateway. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 user/quickstart user/http-routing diff --git a/docs/v0.2.0/about_docs.rst b/docs/v0.2.0/about_docs.rst index 64a12d791d7..ecaa28247b9 100644 --- a/docs/v0.2.0/about_docs.rst +++ b/docs/v0.2.0/about_docs.rst @@ -1,4 +1,4 @@ -About the documentation +About the Documentation ======================= Learn how to contribute to Envoy Gateway documentation. diff --git a/docs/v0.2.0/get_involved.rst b/docs/v0.2.0/get_involved.rst index cd4a70b07ce..f17febd5651 100644 --- a/docs/v0.2.0/get_involved.rst +++ b/docs/v0.2.0/get_involved.rst @@ -1,4 +1,4 @@ -Getting involved +Getting Involved ================ We welcome contributions from the community. Please carefully review the diff --git a/docs/v0.2.0/index.rst b/docs/v0.2.0/index.rst index 0c054b057d0..39f345fa125 100644 --- a/docs/v0.2.0/index.rst +++ b/docs/v0.2.0/index.rst @@ -1,24 +1,19 @@ `Envoy Gateway `_ ============= -Release |version| +Release: |version| .. image:: https://img.shields.io/badge/slack-join-orange.svg :target: https://envoyproxy.slack.com/archives/C03E6NHLESV :alt: Join the Envoy Slack -Envoy Gateway is an open source project for managing Envoy Proxy as a -standalone or Kubernetes-based application gateway. - -.. note:: - - This project is under active development. Many, many features are not - complete. We would love for you to :doc:`get involved`. +Envoy Gateway is an open source project for managing `Envoy Proxy`_ as a standalone or Kubernetes-based application +gateway. `Gateway API`_ resources are used to dynamically provision and configure the managed Envoy Proxies. Whether +you are interested in using or contributing to Envoy Gateway, the following resources will help you get started: .. toctree:: :maxdepth: 1 - intro/index intro/compatibility user_docs design_docs @@ -27,3 +22,11 @@ standalone or Kubernetes-based application gateway. roadmap about_docs get_involved + +.. note:: + + This project is under active development. Many, many features are not + complete. We would love for you to :doc:`get involved`. + +.. _Envoy Proxy: https://www.envoyproxy.io/ +.. _Gateway API: https://gateway-api.sigs.k8s.io/ diff --git a/docs/v0.2.0/intro/index.rst b/docs/v0.2.0/intro/index.rst deleted file mode 100644 index ef349d9d987..00000000000 --- a/docs/v0.2.0/intro/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -Introduction -============ - -Envoy Gateway is an open source project for managing Envoy Proxy as a -standalone or Kubernetes-based application gateway. Currently, it uses -Gateway API as its sole configuration language. - -Many things are in the scope of Envoy Gateway. Many things are not. Many -things (like support for non-Kubernetes instances) will be in scope later, -but are not now. - -.. note:: - - This project is under active development. Many, many features are not - complete. We would love for you to :doc:`get involved<../get_involved>`. diff --git a/docs/v0.2.0/user_docs.rst b/docs/v0.2.0/user_docs.rst index 2f4f77719cf..5a2f83e312e 100644 --- a/docs/v0.2.0/user_docs.rst +++ b/docs/v0.2.0/user_docs.rst @@ -4,7 +4,7 @@ User Guides Learn how to deploy, use, and operate Envoy Gateway. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 user/quickstart user/http-routing From f75158122ad8c4746cd662a94f04ef913e48276d Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Tue, 8 Nov 2022 03:37:38 +0800 Subject: [PATCH 093/113] Explain the non-transparent mode design decision for TCP/UDP (#685) * explain the non-transparent mode design decision for TCP/UDP Signed-off-by: zhaohuabing Co-authored-by: Arko Dasgupta --- docs/latest/design/tcp-udp-design.md | 47 ++++++++++++++++++++++++++++ docs/latest/design_docs.rst | 1 + 2 files changed, 48 insertions(+) create mode 100644 docs/latest/design/tcp-udp-design.md diff --git a/docs/latest/design/tcp-udp-design.md b/docs/latest/design/tcp-udp-design.md new file mode 100644 index 00000000000..691838945af --- /dev/null +++ b/docs/latest/design/tcp-udp-design.md @@ -0,0 +1,47 @@ +# TCP and UDP Proxy Design + +Even though most of the use cases for Envoy Gateway are at Layer-7, Envoy Gateway can also work at Layer-4 to proxy TCP +and UDP traffic. This document will explore the options we have when operating Envoy Gateway at Layer-4 and explain the +design decision. + +Envoy can work as a non-transparent proxy or a transparent proxy for both [TCP](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/other_features/ip_transparency#arch-overview-ip-transparency-original-src-listener) + and [UDP](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto#envoy-v3-api-msg-extensions-filters-udp-udp-proxy-v3-udpproxyconfig) +, so ideally, Envoy Gateway should also be able to work in these two modes: + +## Non-transparent Proxy Mode +For TCP, Envoy terminates the downstream connection, connects the upstream with its own IP address, and proxies the +TCP traffic from the downstream to the upstream. + +For UDP, Envoy receives UDP packages from the downstream, and uses its own IP address as the sender IP address when +proxying the UDP packages to the upstream. + +In this mode, the upstream will see Envoy's IP address. + +## Transparent Proxy Mode +For TCP, Envoy terminates the downstream connection, connects the upstream with the downstream IP address, and proxies +the TCP traffic from the downstream to the upstream. + +For UDP, Envoy receives UDP packages from the downstream, and uses the downstream IP address as the sender IP address +when proxying the UDP packages to the upstream. + +In this mode, the upstream will see the original downstream IP address. + +Note: Even in transparent mode, the upstream can't see the port number of the downstream because Envoy doesn't forward +the port number. + +## The Implications of Transparent Proxy Mode + +### Escalated Privilege +Envoy needs to bind to the downstream IP when connecting to the upstream, which means Envoy requires escalated +CAP_NET_ADMIN privileges. This is often considered as a bad security practice and not allowed in some sensitive deployments. + +### Routing +The upstream can see the original source IP, but the original port number won't be passed, so the return +traffic from the upstream must be routed back to Envoy because only Envoy knows how to send the return traffic back +to the right port number of the downstream, which requires routing at the upstream side to be set up. +In a Kubernetes cluster, Envoy Gateway will have to carefully cooperate with CNI plugins to get the routing right. + +## The Design Decision (For Now) + +The implementation will only support proxying in non-transparent mode i.e. the backend will see the source IP and +port of the deployed Envoy instance instead of the client. diff --git a/docs/latest/design_docs.rst b/docs/latest/design_docs.rst index 4e95a518d1e..82f87e7650b 100644 --- a/docs/latest/design_docs.rst +++ b/docs/latest/design_docs.rst @@ -10,3 +10,4 @@ Learn about the internal details of Envoy Gateway. design/gatewayapi-translator design/watching design/config-api + design/tcp-udp-design \ No newline at end of file From 7c6c37ad72a6fb2403f7ea85b94732c0e14906f7 Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Tue, 8 Nov 2022 03:41:29 +0800 Subject: [PATCH 094/113] add ir for udp route (#646) * add ir for udp route #641 Signed-off-by: zhaohuabing --- internal/ir/xds.go | 54 ++++++++++++++++++++++ internal/ir/xds_test.go | 67 ++++++++++++++++++++++++++++ internal/ir/zz_generated.deepcopy.go | 37 +++++++++++++++ 3 files changed, 158 insertions(+) diff --git a/internal/ir/xds.go b/internal/ir/xds.go index def2bfd5189..bd0656917df 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -43,6 +43,8 @@ type Xds struct { HTTP []*HTTPListener // TCP Listeners exposed by the gateway. TCP []*TCPListener + // UDP Listeners exposed by the gateway. + UDP []*UDPListener } // Validate the fields within the Xds structure. @@ -53,6 +55,16 @@ func (x Xds) Validate() error { errs = multierror.Append(errs, err) } } + for _, tcp := range x.TCP { + if err := tcp.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + for _, udp := range x.UDP { + if err := udp.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } return errs } @@ -74,6 +86,15 @@ func (x Xds) GetTCPListener(name string) *TCPListener { return nil } +func (x Xds) GetUDPListener(name string) *UDPListener { + for _, listener := range x.UDP { + if listener.Name == name { + return listener + } + } + return nil +} + // Printable returns a deep copy of the resource that can be safely logged. func (x Xds) Printable() *Xds { out := x.DeepCopy() @@ -471,3 +492,36 @@ func (t TLSInspectorConfig) Validate() error { } return errs } + +// UDPListener holds the UDP listener configuration. +// +k8s:deepcopy-gen=true +type UDPListener struct { + // Name of the UDPListener + Name string + // Address that the listener should listen on. + Address string + // Port on which the service can be expected to be accessed by clients. + Port uint32 + // Destinations associated with UDP traffic to the service. + Destinations []*RouteDestination +} + +// Validate the fields within the UDPListener structure +func (h UDPListener) Validate() error { + var errs error + if h.Name == "" { + errs = multierror.Append(errs, ErrListenerNameEmpty) + } + if ip := net.ParseIP(h.Address); ip == nil { + errs = multierror.Append(errs, ErrListenerAddressInvalid) + } + if h.Port == 0 { + errs = multierror.Append(errs, ErrListenerPortInvalid) + } + for _, route := range h.Destinations { + if err := route.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs +} diff --git a/internal/ir/xds_test.go b/internal/ir/xds_test.go index 32dd356cd19..cf06dab00dc 100644 --- a/internal/ir/xds_test.go +++ b/internal/ir/xds_test.go @@ -89,6 +89,31 @@ var ( Destinations: []*RouteDestination{&happyRouteDestination}, } + // UDPListener + happyUDPListener = UDPListener{ + Name: "happy", + Address: "0.0.0.0", + Port: 80, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + invalidNameUDPListener = UDPListener{ + Address: "0.0.0.0", + Port: 80, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + invalidAddrUDPListener = UDPListener{ + Name: "invalid-addr", + Address: "1.0.0", + Port: 80, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + invalidPortUDPListenerT = UDPListener{ + Name: "invalid-port", + Address: "0.0.0.0", + Port: 0, + Destinations: []*RouteDestination{&happyRouteDestination}, + } + // HTTPRoute happyHTTPRoute = HTTPRoute{ Name: "happy", @@ -471,6 +496,48 @@ func TestValidateTLSListenerConfig(t *testing.T) { } } +func TestValidateUDPListener(t *testing.T) { + tests := []struct { + name string + input UDPListener + want []error + }{ + { + name: "udp happy", + input: happyUDPListener, + want: nil, + }, + { + name: "udp invalid name", + input: invalidNameUDPListener, + want: []error{ErrListenerNameEmpty}, + }, + { + name: "udp invalid addr", + input: invalidAddrUDPListener, + want: []error{ErrListenerAddressInvalid}, + }, + { + name: "udp invalid port", + input: invalidPortUDPListenerT, + want: []error{ErrListenerPortInvalid}, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + if test.want == nil { + require.NoError(t, test.input.Validate()) + } else { + got := test.input.Validate() + for _, w := range test.want { + assert.ErrorContains(t, got, w.Error()) + } + } + }) + } +} + func TestValidateHTTPRoute(t *testing.T) { tests := []struct { name string diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 0f26e03b505..6458241fdc0 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -439,6 +439,32 @@ func (in *TLSListenerConfig) DeepCopy() *TLSListenerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UDPListener) DeepCopyInto(out *UDPListener) { + *out = *in + if in.Destinations != nil { + in, out := &in.Destinations, &out.Destinations + *out = make([]*RouteDestination, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(RouteDestination) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UDPListener. +func (in *UDPListener) DeepCopy() *UDPListener { + if in == nil { + return nil + } + out := new(UDPListener) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Xds) DeepCopyInto(out *Xds) { *out = *in @@ -464,6 +490,17 @@ func (in *Xds) DeepCopyInto(out *Xds) { } } } + if in.UDP != nil { + in, out := &in.UDP, &out.UDP + *out = make([]*UDPListener, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(UDPListener) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Xds. From 7db9a4084eb14a360660329883eb486897715254 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 8 Nov 2022 10:39:37 +0800 Subject: [PATCH 095/113] feat: support multi-release versions (#698) Signed-off-by: bitliu --- tools/make/docs.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make/docs.mk b/tools/make/docs.mk index dba9d0b4f6e..c2c9b14897e 100644 --- a/tools/make/docs.mk +++ b/tools/make/docs.mk @@ -1,10 +1,10 @@ DOCS_OUTPUT_DIR := docs/html +RELEASE_VERSIONS ?= $(foreach v,$(wildcard ${ROOT_DIR}/docs/*),$(notdir ${v})) ##@ Docs .PHONY: docs docs: docs.clean $(tools/sphinx-build) ## Generate Envoy Gateway Docs Sources - $(eval RELEASE_VERSIONS := latest $(shell cat VERSION)) mkdir -p $(DOCS_OUTPUT_DIR) cp docs/index.html $(DOCS_OUTPUT_DIR)/index.html @for VERSION in $(RELEASE_VERSIONS); do \ From 672a0cc2b926e55fcd67abfa0ba92cb99712c9e8 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Wed, 9 Nov 2022 02:36:06 +0800 Subject: [PATCH 096/113] feat: set envoyproxy image to envoy-dev latest in main (#712) feat: set envoy image to dev latest Signed-off-by: bitliu --- internal/ir/infra.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ir/infra.go b/internal/ir/infra.go index c898d9f881c..488647be21b 100644 --- a/internal/ir/infra.go +++ b/internal/ir/infra.go @@ -16,7 +16,7 @@ import ( const ( DefaultProxyName = "default" - DefaultProxyImage = "envoyproxy/envoy:v1.24-latest" + DefaultProxyImage = "envoyproxy/envoy-dev:latest" ) // Infra defines managed infrastructure. @@ -36,7 +36,7 @@ type ProxyInfra struct { // Config defines user-facing configuration of the managed proxy infrastructure. Config *v1alpha1.EnvoyProxy // Image is the container image used for the managed proxy infrastructure. - // If unset, defaults to "envoyproxy/envoy:v1.24-latest". + // If unset, defaults to "envoyproxy/envoy-dev:latest". Image string // Listeners define the listeners exposed by the proxy infrastructure. Listeners []ProxyListener From 6f800aad01009facbe6dcea90ddf133e93559751 Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Wed, 9 Nov 2022 09:46:03 +0800 Subject: [PATCH 097/113] fix: incorrect level of envoy-gateway configmap vars (#711) * fix: incorrect level of envoy-gateway configmap Signed-off-by: bitliu --- internal/envoygateway/config/decoder.go | 9 +- internal/envoygateway/config/decoder_test.go | 83 ++++++++++++++++++- .../decoder/in/gateway-mixing-provider.yaml | 6 ++ .../decoder/in/invalid-gateway-group.yaml | 6 ++ .../decoder/in/invalid-gateway-kind.yaml | 6 ++ .../decoder/in/invalid-gateway-version.yaml | 6 ++ .../decoder/in/provider-mixing-gateway.yaml | 6 ++ .../decoder/in/provider-with-gateway.yaml | 6 ++ .../config/envoy-gateway/envoy-gateway.yaml | 4 +- 9 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 internal/envoygateway/config/testdata/decoder/in/gateway-mixing-provider.yaml create mode 100644 internal/envoygateway/config/testdata/decoder/in/invalid-gateway-group.yaml create mode 100644 internal/envoygateway/config/testdata/decoder/in/invalid-gateway-kind.yaml create mode 100644 internal/envoygateway/config/testdata/decoder/in/invalid-gateway-version.yaml create mode 100644 internal/envoygateway/config/testdata/decoder/in/provider-mixing-gateway.yaml create mode 100644 internal/envoygateway/config/testdata/decoder/in/provider-with-gateway.yaml diff --git a/internal/envoygateway/config/decoder.go b/internal/envoygateway/config/decoder.go index cb9f0e742d0..636be4cc3df 100644 --- a/internal/envoygateway/config/decoder.go +++ b/internal/envoygateway/config/decoder.go @@ -6,6 +6,7 @@ package config import ( + "errors" "os" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -28,16 +29,16 @@ func Decode(cfgPath string) (*v1alpha1.EnvoyGateway, error) { } // Figure out the resource type from the Group|Version|Kind. - if gvk.Group != v1alpha1.GroupVersion.Group && - gvk.Version != v1alpha1.GroupVersion.Version && + if gvk.Group != v1alpha1.GroupVersion.Group || + gvk.Version != v1alpha1.GroupVersion.Version || gvk.Kind != v1alpha1.KindEnvoyGateway { - return nil, err + return nil, errors.New("failed to decode unmatched resource type") } // Attempt to cast the object. eg, ok := obj.(*v1alpha1.EnvoyGateway) if !ok { - return nil, err + return nil, errors.New("failed to convert object to EnvoyGateway type") } return eg, nil diff --git a/internal/envoygateway/config/decoder_test.go b/internal/envoygateway/config/decoder_test.go index 33b30385ad3..d71748ec61e 100644 --- a/internal/envoygateway/config/decoder_test.go +++ b/internal/envoygateway/config/decoder_test.go @@ -6,6 +6,7 @@ package config import ( + "reflect" "testing" "github.com/stretchr/testify/require" @@ -50,6 +51,74 @@ func TestDecode(t *testing.T) { }, expect: true, }, + { + in: inPath + "provider-with-gateway.yaml", + out: &v1alpha1.EnvoyGateway{ + TypeMeta: metav1.TypeMeta{ + Kind: v1alpha1.KindEnvoyGateway, + APIVersion: v1alpha1.GroupVersion.String(), + }, + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + Provider: v1alpha1.DefaultProvider(), + }, + }, + expect: true, + }, + { + in: inPath + "provider-mixing-gateway.yaml", + out: &v1alpha1.EnvoyGateway{ + TypeMeta: metav1.TypeMeta{ + Kind: v1alpha1.KindEnvoyGateway, + APIVersion: v1alpha1.GroupVersion.String(), + }, + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Provider: v1alpha1.DefaultProvider(), + }, + }, + expect: true, + }, + { + in: inPath + "gateway-mixing-provider.yaml", + out: &v1alpha1.EnvoyGateway{ + TypeMeta: metav1.TypeMeta{ + Kind: v1alpha1.KindEnvoyGateway, + APIVersion: v1alpha1.GroupVersion.String(), + }, + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + }, + }, + expect: true, + }, + { + in: inPath + "provider-mixing-gateway.yaml", + out: &v1alpha1.EnvoyGateway{ + TypeMeta: metav1.TypeMeta{ + Kind: v1alpha1.KindEnvoyGateway, + APIVersion: v1alpha1.GroupVersion.String(), + }, + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Provider: v1alpha1.DefaultProvider(), + Gateway: v1alpha1.DefaultGateway(), + }, + }, + expect: false, + }, + { + in: inPath + "gateway-mixing-provider.yaml", + out: &v1alpha1.EnvoyGateway{ + TypeMeta: metav1.TypeMeta{ + Kind: v1alpha1.KindEnvoyGateway, + APIVersion: v1alpha1.GroupVersion.String(), + }, + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Provider: v1alpha1.DefaultProvider(), + Gateway: v1alpha1.DefaultGateway(), + }, + }, + expect: false, + }, { in: inPath + "no-api-version.yaml", expect: false, @@ -62,6 +131,18 @@ func TestDecode(t *testing.T) { in: "/non/existent/config.yaml", expect: false, }, + { + in: inPath + "invalid-gateway-group.yaml", + expect: false, + }, + { + in: inPath + "invalid-gateway-kind.yaml", + expect: false, + }, + { + in: inPath + "invalid-gateway-version.yaml", + expect: false, + }, } for _, tc := range testCases { @@ -72,7 +153,7 @@ func TestDecode(t *testing.T) { require.NoError(t, err) require.Equal(t, tc.out, eg) } else { - require.Error(t, err, "An error was expected") + require.Equal(t, (!reflect.DeepEqual(tc.out, eg) || err != nil), true) } }) } diff --git a/internal/envoygateway/config/testdata/decoder/in/gateway-mixing-provider.yaml b/internal/envoygateway/config/testdata/decoder/in/gateway-mixing-provider.yaml new file mode 100644 index 00000000000..aaef7204554 --- /dev/null +++ b/internal/envoygateway/config/testdata/decoder/in/gateway-mixing-provider.yaml @@ -0,0 +1,6 @@ +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + provider: + type: Kubernetes diff --git a/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-group.yaml b/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-group.yaml new file mode 100644 index 00000000000..d38e2abbc37 --- /dev/null +++ b/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-group.yaml @@ -0,0 +1,6 @@ +apiVersion: configs.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: Kubernetes +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-kind.yaml b/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-kind.yaml new file mode 100644 index 00000000000..9fe60c708b4 --- /dev/null +++ b/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-kind.yaml @@ -0,0 +1,6 @@ +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateways +provider: + type: Kubernetes +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-version.yaml b/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-version.yaml new file mode 100644 index 00000000000..74b64c923ac --- /dev/null +++ b/internal/envoygateway/config/testdata/decoder/in/invalid-gateway-version.yaml @@ -0,0 +1,6 @@ +apiVersion: config.gateway.envoyproxy.io/v1beta +kind: EnvoyGateway +provider: + type: Kubernetes +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/internal/envoygateway/config/testdata/decoder/in/provider-mixing-gateway.yaml b/internal/envoygateway/config/testdata/decoder/in/provider-mixing-gateway.yaml new file mode 100644 index 00000000000..128d8f89f8f --- /dev/null +++ b/internal/envoygateway/config/testdata/decoder/in/provider-mixing-gateway.yaml @@ -0,0 +1,6 @@ +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: Kubernetes + gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/internal/envoygateway/config/testdata/decoder/in/provider-with-gateway.yaml b/internal/envoygateway/config/testdata/decoder/in/provider-with-gateway.yaml new file mode 100644 index 00000000000..86be0afa6f2 --- /dev/null +++ b/internal/envoygateway/config/testdata/decoder/in/provider-with-gateway.yaml @@ -0,0 +1,6 @@ +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: Kubernetes +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller diff --git a/internal/provider/kubernetes/config/envoy-gateway/envoy-gateway.yaml b/internal/provider/kubernetes/config/envoy-gateway/envoy-gateway.yaml index 128d8f89f8f..86be0afa6f2 100644 --- a/internal/provider/kubernetes/config/envoy-gateway/envoy-gateway.yaml +++ b/internal/provider/kubernetes/config/envoy-gateway/envoy-gateway.yaml @@ -2,5 +2,5 @@ apiVersion: config.gateway.envoyproxy.io/v1alpha1 kind: EnvoyGateway provider: type: Kubernetes - gateway: - controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller From 0446430e01742b06be48446394def5424be977ca Mon Sep 17 00:00:00 2001 From: zirain Date: Wed, 9 Nov 2022 09:57:18 +0800 Subject: [PATCH 098/113] translator: add accesslog (#704) * translator: add accesslog Signed-off-by: hejianpeng --- internal/xds/translator/accesslog.go | 24 +++++++++++++ internal/xds/translator/listener.go | 32 +++++++++++++++++ .../http-route-direct-response.listeners.yaml | 16 ++++++++- .../xds-ir/http-route-redirect.listeners.yaml | 16 ++++++++- .../http-route-request-headers.listeners.yaml | 16 ++++++++- ...te-weighted-invalid-backend.listeners.yaml | 16 ++++++++- .../out/xds-ir/http-route.listeners.yaml | 16 ++++++++- ...ultiple-listeners-same-port.listeners.yaml | 36 ++++++++++++++++++- .../out/xds-ir/simple-tls.listeners.yaml | 16 ++++++++- .../tls-route-passthrough.listeners.yaml | 16 ++++++++- 10 files changed, 196 insertions(+), 8 deletions(-) create mode 100644 internal/xds/translator/accesslog.go diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go new file mode 100644 index 00000000000..82b6ef95b05 --- /dev/null +++ b/internal/xds/translator/accesslog.go @@ -0,0 +1,24 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" + fileaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3" +) + +var ( + stdoutFileAccessLog = &fileaccesslog.FileAccessLog{ + Path: "/dev/stdout", + } + + // for the case when a route does not exist to upstream, hcm logs will not be present + listenerAccessLogFilter = &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_ResponseFlagFilter{ + ResponseFlagFilter: &accesslog.ResponseFlagFilter{Flags: []string{"NR"}}, + }, + } +) diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index e811f22db1f..0e254d06b18 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -8,6 +8,7 @@ package translator import ( "errors" + accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" router "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" @@ -22,8 +23,16 @@ import ( ) func buildXdsListener(name, address string, port uint32) *listener.Listener { + accesslogAny, _ := anypb.New(stdoutFileAccessLog) return &listener.Listener{ Name: name, + AccessLog: []*accesslog.AccessLog{ + { + Name: wellknown.FileAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{TypedConfig: accesslogAny}, + Filter: listenerAccessLogFilter, + }, + }, Address: &core.Address{ Address: &core.Address_SocketAddress{ SocketAddress: &core.SocketAddress{ @@ -44,6 +53,11 @@ func addXdsHTTPFilterChain(xdsListener *listener.Listener, irListener *ir.HTTPLi return err } + accesslogAny, err := anypb.New(stdoutFileAccessLog) + if err != nil { + return err + } + // HTTP filter configuration var statPrefix string if irListener.TLS != nil { @@ -52,6 +66,12 @@ func addXdsHTTPFilterChain(xdsListener *listener.Listener, irListener *ir.HTTPLi statPrefix = "http" } mgr := &hcm.HttpConnectionManager{ + AccessLog: []*accesslog.AccessLog{ + { + Name: wellknown.FileAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{TypedConfig: accesslogAny}, + }, + }, CodecType: hcm.HttpConnectionManager_AUTO, StatPrefix: statPrefix, RouteSpecifier: &hcm.HttpConnectionManager_Rds{ @@ -153,7 +173,19 @@ func addXdsTCPFilterChain(xdsListener *listener.Listener, irListener *ir.TCPList if irListener.TLS != nil { statPrefix = "passthrough" } + + accesslogAny, err := anypb.New(stdoutFileAccessLog) + if err != nil { + return err + } + mgr := &tcp.TcpProxy{ + AccessLog: []*accesslog.AccessLog{ + { + Name: wellknown.FileAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{TypedConfig: accesslogAny}, + }, + }, StatPrefix: statPrefix, ClusterSpecifier: &tcp.TcpProxy_Cluster{ Cluster: clusterName, diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml index 2f73c8e922c..6c555573979 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-direct-response.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml index 2f73c8e922c..6c555573979 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-redirect.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml index 2f73c8e922c..6c555573979 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-request-headers.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml index 2f73c8e922c..6c555573979 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml index 2f73c8e922c..6c555573979 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: diff --git a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml index 445e851caa2..e44df68af1f 100644 --- a/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/multiple-listeners-same-port.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: @@ -31,6 +45,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: @@ -70,6 +89,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: @@ -109,6 +133,11 @@ - name: envoy.filters.network.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout cluster: fifth-listener statPrefix: passthrough - filterChainMatch: @@ -118,6 +147,11 @@ - name: envoy.filters.network.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout cluster: sixth-listener statPrefix: passthrough listenerFilters: diff --git a/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml index e98b44a0194..1a3e22e469f 100644 --- a/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/simple-tls.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -7,6 +16,11 @@ - name: envoy.filters.network.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout httpFilters: - name: envoy.filters.http.router typedConfig: diff --git a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml index 53774a46925..2186e91d327 100644 --- a/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/tls-route-passthrough.listeners.yaml @@ -1,4 +1,13 @@ -- address: +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: socketAddress: address: 0.0.0.0 portValue: 10080 @@ -10,6 +19,11 @@ - name: envoy.filters.network.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout cluster: tls-passthrough statPrefix: passthrough listenerFilters: From 9eecce661715577f9a95b0ec28d9a7450716d715 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Tue, 8 Nov 2022 18:58:46 -0800 Subject: [PATCH 099/113] Update roadmap for v0.3.0 (#695) * Update roadmap for v0.3.0 Signed-off-by: Arko Dasgupta --- docs/latest/design/roadmap.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/latest/design/roadmap.md b/docs/latest/design/roadmap.md index d6ec649e4a2..ee219500bf8 100644 --- a/docs/latest/design/roadmap.md +++ b/docs/latest/design/roadmap.md @@ -22,7 +22,7 @@ Roadmap features and timelines may change based on feedback, community contribut roadmap item, you're encouraged to attend a community meeting to discuss the details, or help us deliver the feature by contributing to the project. -`Last Updated: October 2022` +`Last Updated: November 2022` ### [v0.2.0][v0.2.0]: Establish a Solid Foundation @@ -34,11 +34,17 @@ contributing to the project. ### [v0.3.0][v0.3.0]: Drive Advanced Features through Extension Mechanisms -- Global Rate Limiting -- AuthN/AuthZ- [Issue #336][336]. -- Lets Encrypt Integration +- Support extended Gateway API fields [Issue #707][707]. +- Support experimental Gateway APIs such as TCPRoute [Issue #643][643], UDPRoute [Issue #641][641] and GRPCRoute [Issue #642][642]. +- Establish guidelines for leveragaing Gateway API extensions [Issue #675][675]. +- Rate Limiting [Issue #670][670]. +- Authentication [Issue #336][336]. -### [v0.4.0][v0.4.0]: Manageability and Scale +### [v0.4.0][v0.4.0]: More Advanced Features through Extension Mechanisms + +- Allow users to configure xDS Resources [Issue #24][24]. + +### [v0.5.0][v0.5.0]: Manageability and Scale - Tooling for devs/infra admins to aid in managing/maintaining EG - Support advanced provisioning use cases (e.g. multi-cluster, serverless, etc.) @@ -52,9 +58,17 @@ contributing to the project. [v0.2.0]: https://github.com/envoyproxy/gateway/milestone/1 [v0.3.0]: https://github.com/envoyproxy/gateway/milestone/7 [v0.4.0]: https://github.com/envoyproxy/gateway/milestone/12 +[v0.5.0]: https://github.com/envoyproxy/gateway/milestone/13 +[17]: https://github.com/envoyproxy/gateway/issues/17 +[24]: https://github.com/envoyproxy/gateway/issues/24 [60]: https://github.com/envoyproxy/gateway/issues/60 +[63]: https://github.com/envoyproxy/gateway/issues/63 [64]: https://github.com/envoyproxy/gateway/issues/64 -[17]: https://github.com/envoyproxy/gateway/issues/17 [65]: https://github.com/envoyproxy/gateway/issues/65 -[63]: https://github.com/envoyproxy/gateway/issues/63 [336]: https://github.com/envoyproxy/gateway/issues/336 +[641]: https://github.com/envoyproxy/gateway/issues/641 +[642]: https://github.com/envoyproxy/gateway/issues/642 +[643]: https://github.com/envoyproxy/gateway/issues/643 +[670]: https://github.com/envoyproxy/gateway/issues/670 +[675]: https://github.com/envoyproxy/gateway/issues/675 +[707]: https://github.com/envoyproxy/gateway/issues/707 From 13875fa33b6ea3203e7d4d0867555da460046cbf Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 9 Nov 2022 08:43:56 -0800 Subject: [PATCH 100/113] Enhance HTTP IR to support GRPCRoute (#666) Relates to https://github.com/envoyproxy/gateway/issues/642 Signed-off-by: Arko Dasgupta --- internal/ir/xds.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/ir/xds.go b/internal/ir/xds.go index bd0656917df..7afee1b39d5 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -123,6 +123,8 @@ type HTTPListener struct { TLS *TLSListenerConfig // Routes associated with HTTP traffic to the service. Routes []*HTTPRoute + // IsHTTP2 is set if the upstream client as well as the downstream server are configured to serve HTTP2 traffic. + IsHTTP2 bool } // Validate the fields within the HTTPListener structure From 2a3c99558c34851519b00b42942a7f5c968830c3 Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Thu, 10 Nov 2022 01:19:11 +0800 Subject: [PATCH 101/113] xds translator for udp route (#709) * xds translator for udp route Signed-off-by: zhaohuabing --- internal/xds/translator/listener.go | 79 ++++++++++++++++++- .../testdata/in/xds-ir/udp-route.yaml | 9 +++ .../out/xds-ir/udp-route.clusters.yaml | 23 ++++++ .../out/xds-ir/udp-route.listeners.yaml | 28 +++++++ .../testdata/out/xds-ir/udp-route.routes.yaml | 1 + internal/xds/translator/translator.go | 33 ++++++-- internal/xds/translator/translator_test.go | 3 + 7 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 internal/xds/translator/testdata/in/xds-ir/udp-route.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/udp-route.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/udp-route.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/udp-route.routes.yaml diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index 0e254d06b18..7d63575ddf5 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -8,6 +8,8 @@ package translator import ( "errors" + xdscore "github.com/cncf/xds/go/xds/core/v3" + matcher "github.com/cncf/xds/go/xds/type/matcher/v3" accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" @@ -15,6 +17,7 @@ import ( tls_inspector "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/tls_inspector/v3" hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tcp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" + udp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3" tls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" "google.golang.org/protobuf/types/known/anypb" @@ -22,7 +25,7 @@ import ( "github.com/envoyproxy/gateway/internal/ir" ) -func buildXdsListener(name, address string, port uint32) *listener.Listener { +func buildXdsTCPListener(name, address string, port uint32) *listener.Listener { accesslogAny, _ := anypb.New(stdoutFileAccessLog) return &listener.Listener{ Name: name, @@ -286,3 +289,77 @@ func buildXdsDownstreamTLSSecret(listenerName string, }, }, nil } + +func buildXdsUDPListener(clusterName string, udpListener *ir.UDPListener) (*listener.Listener, error) { + if udpListener == nil { + return nil, errors.New("udp listener is nil") + } + + statPrefix := "service" + + route := &udp.Route{ + Cluster: clusterName, + } + routeAny, err := anypb.New(route) + if err != nil { + return nil, err + } + accesslogAny, _ := anypb.New(stdoutFileAccessLog) + udpProxy := &udp.UdpProxyConfig{ + StatPrefix: statPrefix, + AccessLog: []*accesslog.AccessLog{ + { + Name: wellknown.FileAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{TypedConfig: accesslogAny}, + }, + }, + RouteSpecifier: &udp.UdpProxyConfig_Matcher{ + Matcher: &matcher.Matcher{ + OnNoMatch: &matcher.Matcher_OnMatch{ + OnMatch: &matcher.Matcher_OnMatch_Action{ + Action: &xdscore.TypedExtensionConfig{ + TypedConfig: routeAny, + }, + }, + }, + }, + }, + } + udpProxyAny, err := anypb.New(udpProxy) + if err != nil { + return nil, err + } + + filterChain := &listener.FilterChain{ + Filters: []*listener.Filter{{ + Name: "envoy.filters.udp_listener.udp_proxy", + ConfigType: &listener.Filter_TypedConfig{ + TypedConfig: udpProxyAny, + }, + }}, + } + + xdsListener := &listener.Listener{ + Name: udpListener.Name, + AccessLog: []*accesslog.AccessLog{ + { + Name: wellknown.FileAccessLog, + ConfigType: &accesslog.AccessLog_TypedConfig{TypedConfig: accesslogAny}, + }, + }, + Address: &core.Address{ + Address: &core.Address_SocketAddress{ + SocketAddress: &core.SocketAddress{ + Protocol: core.SocketAddress_UDP, + Address: udpListener.Address, + PortSpecifier: &core.SocketAddress_PortValue{ + PortValue: udpListener.Port, + }, + }, + }, + }, + FilterChains: []*listener.FilterChain{filterChain}, + } + + return xdsListener, nil +} diff --git a/internal/xds/translator/testdata/in/xds-ir/udp-route.yaml b/internal/xds/translator/testdata/in/xds-ir/udp-route.yaml new file mode 100644 index 00000000000..490b4aa2121 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/udp-route.yaml @@ -0,0 +1,9 @@ +udp: +- name: "udp-route" + address: "0.0.0.0" + port: 10080 + destinations: + - host: "1.2.3.4" + port: 50000 + - host: "5.6.7.8" + port: 50001 diff --git a/internal/xds/translator/testdata/out/xds-ir/udp-route.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/udp-route.clusters.yaml new file mode 100644 index 00000000000..3201aa8be50 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/udp-route.clusters.yaml @@ -0,0 +1,23 @@ +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: udp-route + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + - endpoint: + address: + socketAddress: + address: 5.6.7.8 + portValue: 50001 + loadBalancingWeight: 1 + locality: {} + name: udp-route + outlierDetection: {} + type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/udp-route.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/udp-route.listeners.yaml new file mode 100644 index 00000000000..d0def54c67b --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/udp-route.listeners.yaml @@ -0,0 +1,28 @@ +- accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + protocol: UDP + filterChains: + - filters: + - name: envoy.filters.udp_listener.udp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + matcher: + onNoMatch: + action: + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: udp-route + statPrefix: service + name: udp-route diff --git a/internal/xds/translator/testdata/out/xds-ir/udp-route.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/udp-route.routes.yaml new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/udp-route.routes.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 7d6f45629ba..626826c655a 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -31,9 +31,9 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { var xdsRouteCfg *route.RouteConfiguration // Search for an existing listener, if it does not exist, create one. - xdsListener := findXdsListener(tCtx, httpListener.Address, httpListener.Port) + xdsListener := findXdsListener(tCtx, httpListener.Address, httpListener.Port, core.SocketAddress_TCP) if xdsListener == nil { - xdsListener = buildXdsListener(httpListener.Name, httpListener.Address, httpListener.Port) + xdsListener = buildXdsTCPListener(httpListener.Name, httpListener.Address, httpListener.Port) tCtx.AddXdsResource(resource.ListenerType, xdsListener) } else if httpListener.TLS == nil { // Find the route config associated with this listener that @@ -113,9 +113,9 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { tCtx.AddXdsResource(resource.ClusterType, xdsCluster) // Search for an existing listener, if it does not exist, create one. - xdsListener := findXdsListener(tCtx, tcpListener.Address, tcpListener.Port) + xdsListener := findXdsListener(tCtx, tcpListener.Address, tcpListener.Port, core.SocketAddress_TCP) if xdsListener == nil { - xdsListener = buildXdsListener(tcpListener.Name, tcpListener.Address, tcpListener.Port) + xdsListener = buildXdsTCPListener(tcpListener.Name, tcpListener.Address, tcpListener.Port) tCtx.AddXdsResource(resource.ListenerType, xdsListener) } @@ -123,11 +123,29 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { return nil, err } } + + for _, udpListener := range ir.UDP { + // 1:1 between IR UDPListener and xDS Cluster + xdsCluster, err := buildXdsCluster(udpListener.Name, udpListener.Destinations) + if err != nil { + return nil, multierror.Append(err, errors.New("error building xds cluster")) + } + tCtx.AddXdsResource(resource.ClusterType, xdsCluster) + + // There won't be multiple UDP listeners on the same port since it's already been checked at the gateway api + // translator + xdsListener, err := buildXdsUDPListener(xdsCluster.Name, udpListener) + if err != nil { + return nil, multierror.Append(err, errors.New("error building xds cluster")) + } + tCtx.AddXdsResource(resource.ListenerType, xdsListener) + } return tCtx, nil } -// findXdsListener finds an xds listener with the same address and port, and returns nil if there is no match. -func findXdsListener(tCtx *types.ResourceVersionTable, address string, port uint32) *listener.Listener { +// findXdsListener finds a xds listener with the same address, port and protocol, and returns nil if there is no match. +func findXdsListener(tCtx *types.ResourceVersionTable, address string, port uint32, + protocol core.SocketAddress_Protocol) *listener.Listener { if tCtx == nil || tCtx.XdsResources == nil || tCtx.XdsResources[resource.ListenerType] == nil { return nil } @@ -135,7 +153,8 @@ func findXdsListener(tCtx *types.ResourceVersionTable, address string, port uint for _, r := range tCtx.XdsResources[resource.ListenerType] { listener := r.(*listener.Listener) addr := listener.GetAddress() - if addr.GetSocketAddress().GetPortValue() == port && addr.GetSocketAddress().Address == address { + if addr.GetSocketAddress().GetPortValue() == port && addr.GetSocketAddress().Address == address && addr. + GetSocketAddress().Protocol == protocol { return listener } } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index ea61d193770..6ed5dd282c9 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -62,6 +62,9 @@ func TestTranslate(t *testing.T) { name: "multiple-listeners-same-port", requireSecrets: true, }, + { + name: "udp-route", + }, } for _, tc := range testCases { From b7502ce3153c7d9f11d334b41d222dfcba77e6a7 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 9 Nov 2022 09:26:49 -0800 Subject: [PATCH 102/113] Add new Xds IR TCP Listener per TLSRoute (#696) * had to also append the TLSRoute name to the listener to make it unique Fixes: https://github.com/envoyproxy/gateway/issues/691 Signed-off-by: Arko Dasgupta --- ...ith-tls-terminate-and-passthrough.out.yaml | 2 +- .../tlsroute-attaching-to-gateway.out.yaml | 2 +- .../testdata/tlsroute-multiple.in.yaml | 58 +++++++++ .../testdata/tlsroute-multiple.out.yaml | 121 ++++++++++++++++++ ...her-namespace-allowed-by-refgrant.out.yaml | 2 +- .../tlsroute-with-empty-hostname.out.yaml | 2 +- ...oute-with-empty-listener-hostname.out.yaml | 2 +- internal/gatewayapi/translator.go | 39 +++--- 8 files changed, 204 insertions(+), 24 deletions(-) create mode 100644 internal/gatewayapi/testdata/tlsroute-multiple.in.yaml create mode 100644 internal/gatewayapi/testdata/tlsroute-multiple.out.yaml diff --git a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml index 26002751f1e..53a3a73bd94 100644 --- a/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-listener-with-tls-terminate-and-passthrough.out.yaml @@ -122,7 +122,7 @@ xdsIR: port: 8080 weight: 1 tcp: - - name: envoy-gateway-gateway-1-tls-passthrough + - name: envoy-gateway-gateway-1-tls-passthrough-tlsroute-1 address: 0.0.0.0 port: 10090 tls: diff --git a/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml index 63eeb40cacd..a6d52c33719 100644 --- a/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-attaching-to-gateway.out.yaml @@ -56,7 +56,7 @@ tlsRoutes: xdsIR: envoy-gateway-gateway-1: tcp: - - name: envoy-gateway-gateway-1-tls + - name: envoy-gateway-gateway-1-tls-tlsroute-1 address: 0.0.0.0 port: 10090 tls: diff --git a/internal/gatewayapi/testdata/tlsroute-multiple.in.yaml b/internal/gatewayapi/testdata/tlsroute-multiple.in.yaml new file mode 100644 index 00000000000..f7c756f0cca --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-multiple.in.yaml @@ -0,0 +1,58 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + hostnames: + - "foo.com" + rules: + - backendRefs: + - name: service-1 + port: 8080 +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-2 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + hostnames: + - "bar.com" + rules: + - backendRefs: + - name: service-1 + port: 8080 +services: +- apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-1 + spec: + clusterIP: 7.7.7.7 + ports: + - port: 8080 diff --git a/internal/gatewayapi/testdata/tlsroute-multiple.out.yaml b/internal/gatewayapi/testdata/tlsroute-multiple.out.yaml new file mode 100644 index 00000000000..6eba7c09502 --- /dev/null +++ b/internal/gatewayapi/testdata/tlsroute-multiple.out.yaml @@ -0,0 +1,121 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1beta1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: tls + protocol: TLS + port: 91 + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All + status: + listeners: + - name: tls + supportedKinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + attachedRoutes: 2 + conditions: + - type: Ready + status: "True" + reason: Ready + message: Listener is ready +tlsRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + hostnames: + - foo.com + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: TLSRoute + metadata: + namespace: default + name: tlsroute-2 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + hostnames: + - bar.com + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - parentRef: + namespace: envoy-gateway + name: gateway-1 + controllerName: gateway.envoyproxy.io/gatewayclass-controller + conditions: + - type: Accepted + status: "True" + reason: Accepted + message: Route is accepted +xdsIR: + envoy-gateway-gateway-1: + tcp: + - name: envoy-gateway-gateway-1-tls-tlsroute-1 + address: 0.0.0.0 + port: 10091 + tls: + snis: + - foo.com + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 + - name: envoy-gateway-gateway-1-tls-tlsroute-2 + address: 0.0.0.0 + port: 10091 + tls: + snis: + - bar.com + destinations: + - host: 7.7.7.7 + port: 8080 + weight: 1 +infraIR: + envoy-gateway-gateway-1: + proxy: + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway-gateway-1 + image: envoyproxy/envoy:translator-tests + listeners: + - address: "" + ports: + - name: tls + protocol: "TLS" + servicePort: 91 + containerPort: 10091 diff --git a/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml index 65926af88cb..16acfdeaffd 100644 --- a/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-with-backendref-in-other-namespace-allowed-by-refgrant.out.yaml @@ -57,7 +57,7 @@ tlsRoutes: xdsIR: envoy-gateway-gateway-1: tcp: - - name: envoy-gateway-gateway-1-tls + - name: envoy-gateway-gateway-1-tls-tlsroute-1 address: 0.0.0.0 port: 10090 tls: diff --git a/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml index 0b5afbd16e1..68cea3fb8d4 100644 --- a/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-with-empty-hostname.out.yaml @@ -55,7 +55,7 @@ tlsRoutes: xdsIR: envoy-gateway-gateway-1: tcp: - - name: envoy-gateway-gateway-1-tls + - name: envoy-gateway-gateway-1-tls-tlsroute-1 address: 0.0.0.0 port: 10091 tls: diff --git a/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml b/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml index eff4d035480..3c086a72820 100644 --- a/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml +++ b/internal/gatewayapi/testdata/tlsroute-with-empty-listener-hostname.out.yaml @@ -57,7 +57,7 @@ tlsRoutes: xdsIR: envoy-gateway-gateway-1: tcp: - - name: envoy-gateway-gateway-1-tls + - name: envoy-gateway-gateway-1-tls-tlsroute-1 address: 0.0.0.0 port: 10091 tls: diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index b7011896931..0d60adc35e9 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -552,7 +552,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap switch listener.Protocol { case v1beta1.HTTPProtocolType, v1beta1.HTTPSProtocolType: irListener := &ir.HTTPListener{ - Name: irListenerName(listener), + Name: irHTTPListenerName(listener), Address: "0.0.0.0", Port: uint32(containerPort), TLS: irTLSConfig(listener.tlsSecret), @@ -566,18 +566,6 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR XdsIRMap irListener.Hostnames = append(irListener.Hostnames, "*") } gwXdsIR.HTTP = append(gwXdsIR.HTTP, irListener) - case v1beta1.TLSProtocolType: - irListener := &ir.TCPListener{ - Name: irListenerName(listener), - Address: "0.0.0.0", - Port: uint32(containerPort), - TLS: &ir.TLSInspectorConfig{ - // Since we need to first compute intersecting hostnames between the - // listener and TLS Route, populate this field after processing TLS Routes. - SNIs: []string{}, - }, - } - gwXdsIR.TCP = append(gwXdsIR.TCP, irListener) } // Add the listener to the Infra IR. Infra IR ports must have a unique port number. @@ -1200,7 +1188,7 @@ func (t *Translator) ProcessHTTPRoutes(httpRoutes []*v1beta1.HTTPRoute, gateways } irKey := irStringKey(listener.gateway) - irListener := xdsIR[irKey].GetHTTPListener(irListenerName(listener)) + irListener := xdsIR[irKey].GetHTTPListener(irHTTPListenerName(listener)) if irListener != nil { irListener.Routes = append(irListener.Routes, perHostRoutes...) } @@ -1385,11 +1373,20 @@ func (t *Translator) ProcessTLSRoutes(tlsRoutes []*v1alpha2.TLSRoute, gateways [ hasHostnameIntersection = true irKey := irStringKey(listener.gateway) - irListener := xdsIR[irKey].GetTCPListener(irListenerName(listener)) - if irListener != nil { - irListener.Destinations = routeDestinations - irListener.TLS.SNIs = hosts + containerPort := servicePortToContainerPort(int32(listener.Port)) + // Create the TCP Listener while parsing the TLSRoute since + // the listener directly links to a routeDestination. + irListener := &ir.TCPListener{ + Name: irTCPListenerName(listener, tlsRoute), + Address: "0.0.0.0", + Port: uint32(containerPort), + TLS: &ir.TLSInspectorConfig{ + SNIs: hosts, + }, + Destinations: routeDestinations, } + gwXdsIR := xdsIR[irKey] + gwXdsIR.TCP = append(gwXdsIR.TCP, irListener) // Theoretically there should only be one parent ref per // Route that attaches to a given Listener, so fine to just increment here, but we @@ -1571,10 +1568,14 @@ func irStringKey(gateway *v1beta1.Gateway) string { return fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name) } -func irListenerName(listener *ListenerContext) string { +func irHTTPListenerName(listener *ListenerContext) string { return fmt.Sprintf("%s-%s-%s", listener.gateway.Namespace, listener.gateway.Name, listener.Name) } +func irTCPListenerName(listener *ListenerContext, tlsRoute *TLSRouteContext) string { + return fmt.Sprintf("%s-%s-%s-%s", listener.gateway.Namespace, listener.gateway.Name, listener.Name, tlsRoute.Name) +} + func routeName(route RouteContext, ruleIdx, matchIdx int) string { return fmt.Sprintf("%s-%s-rule-%d-match-%d", route.GetNamespace(), route.GetName(), ruleIdx, matchIdx) } From bb2df7c7d195f4221f894b0e07a515690d4e3947 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 9 Nov 2022 10:36:48 -0800 Subject: [PATCH 103/113] Updates Readme Slack and Google Group (#715) Updates Readme Slack and Google Group Signed-off-by: danehans --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6f6e80f993e..d750c980ba1 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Kubernetes-based application gateway. ## Contact -* Slack: Go to [Envoy Gateway Channel](https://envoyproxy.slack.com/archives/C03E6NHLESV). -* [Google Group][group]: Envoy Gateway developer discussion (APIs, feature design, etc.). +* Slack: Join the [Envoy Slack workspace][] if you're not already a member. Otherwise, use the + [Envoy Gateway channel][] to start collaborating with the community. ## Contributing @@ -33,3 +33,5 @@ The Envoy Gateway team meets every Tuesday and Thursday. Refer to the meeting de [meeting]: https://docs.google.com/document/d/1leqwsHX8N-XxNEyTflYjRur462ukFxd19Rnk3Uzy55I/edit?usp=sharing [group]: https://groups.google.com/forum/#!forum/envoy-gateway-developers [blog]: https://blog.envoyproxy.io/introducing-envoy-gateway-ad385cc59532 +[Envoy Slack workspace]: https://communityinviter.com/apps/envoyproxy/envoy +[Envoy Gateway channel]: https://envoyproxy.slack.com/archives/C03E6NHLESV From ee75e3b21d116b55929000c783211b657d9ab0a5 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Wed, 9 Nov 2022 14:33:04 -0800 Subject: [PATCH 104/113] add xds translation for http2 (#718) Relates to https://github.com/envoyproxy/gateway/issues/642 Signed-off-by: Arko Dasgupta --- internal/xds/translator/cluster.go | 12 ++++-- .../testdata/in/xds-ir/http2-route.yaml | 22 ++++++++++ .../out/xds-ir/http2-route.clusters.yaml | 19 +++++++++ .../out/xds-ir/http2-route.listeners.yaml | 40 +++++++++++++++++++ .../out/xds-ir/http2-route.routes.yaml | 18 +++++++++ internal/xds/translator/translator.go | 6 +-- internal/xds/translator/translator_test.go | 3 ++ 7 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 internal/xds/translator/testdata/in/xds-ir/http2-route.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http2-route.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http2-route.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http2-route.routes.yaml diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index 4f0c479f983..df91a5df124 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -17,7 +17,7 @@ import ( "github.com/envoyproxy/gateway/internal/ir" ) -func buildXdsCluster(routeName string, destinations []*ir.RouteDestination) (*cluster.Cluster, error) { +func buildXdsCluster(routeName string, destinations []*ir.RouteDestination, isHTTP2 bool) (*cluster.Cluster, error) { localities := make([]*endpoint.LocalityLbEndpoints, 0, 1) locality := &endpoint.LocalityLbEndpoints{ Locality: &core.Locality{}, @@ -29,7 +29,7 @@ func buildXdsCluster(routeName string, destinations []*ir.RouteDestination) (*cl LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}} localities = append(localities, locality) clusterName := routeName - return &cluster.Cluster{ + cluster := &cluster.Cluster{ Name: clusterName, ConnectTimeout: durationpb.New(5 * time.Second), ClusterDiscoveryType: &cluster.Cluster_Type{Type: cluster.Cluster_STATIC}, @@ -40,7 +40,13 @@ func buildXdsCluster(routeName string, destinations []*ir.RouteDestination) (*cl LocalityConfigSpecifier: &cluster.Cluster_CommonLbConfig_LocalityWeightedLbConfig_{ LocalityWeightedLbConfig: &cluster.Cluster_CommonLbConfig_LocalityWeightedLbConfig{}}}, OutlierDetection: &cluster.OutlierDetection{}, - }, nil + } + + if isHTTP2 { + cluster.Http2ProtocolOptions = &core.Http2ProtocolOptions{} + } + + return cluster, nil } diff --git a/internal/xds/translator/testdata/in/xds-ir/http2-route.yaml b/internal/xds/translator/testdata/in/xds-ir/http2-route.yaml new file mode 100644 index 00000000000..131d775c9bf --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http2-route.yaml @@ -0,0 +1,22 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + isHTTP2: true + routes: + - name: "first-route" + pathMatch: + name: "test" + exact: "foo/bar" + headerMatches: + - name: user + stringMatch: + exact: "jason" + queryParamMatches: + - name: "debug" + exact: "yes" + destinations: + - host: "1.2.3.4" + port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/http2-route.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http2-route.clusters.yaml new file mode 100644 index 00000000000..3c1acd4f3a6 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http2-route.clusters.yaml @@ -0,0 +1,19 @@ +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 5s + dnsLookupFamily: V4_ONLY + http2ProtocolOptions: {} + loadAssignment: + clusterName: first-route + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + locality: {} + name: first-route + outlierDetection: {} + type: STATIC diff --git a/internal/xds/translator/testdata/out/xds-ir/http2-route.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http2-route.listeners.yaml new file mode 100644 index 00000000000..6c555573979 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http2-route.listeners.yaml @@ -0,0 +1,40 @@ +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + rds: + configSource: + apiConfigSource: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: xds_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + resourceApiVersion: V3 + routeConfigName: first-listener + statPrefix: http + name: first-listener diff --git a/internal/xds/translator/testdata/out/xds-ir/http2-route.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http2-route.routes.yaml new file mode 100644 index 00000000000..a28539a6d49 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http2-route.routes.yaml @@ -0,0 +1,18 @@ +- name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener + routes: + - match: + headers: + - name: user + stringMatch: + exact: jason + path: foo/bar + queryParameters: + - name: debug + stringMatch: + exact: "yes" + route: + cluster: first-route diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 626826c655a..eeaf4a192e1 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -93,7 +93,7 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { if len(httpRoute.Destinations) == 0 && httpRoute.BackendWeights.Invalid > 0 { continue } - xdsCluster, err := buildXdsCluster(httpRoute.Name, httpRoute.Destinations) + xdsCluster, err := buildXdsCluster(httpRoute.Name, httpRoute.Destinations, httpListener.IsHTTP2) if err != nil { return nil, multierror.Append(err, errors.New("error building xds cluster")) } @@ -106,7 +106,7 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { for _, tcpListener := range ir.TCP { // 1:1 between IR TCPListener and xDS Cluster - xdsCluster, err := buildXdsCluster(tcpListener.Name, tcpListener.Destinations) + xdsCluster, err := buildXdsCluster(tcpListener.Name, tcpListener.Destinations, false /*isHTTP2 */) if err != nil { return nil, multierror.Append(err, errors.New("error building xds cluster")) } @@ -126,7 +126,7 @@ func Translate(ir *ir.Xds) (*types.ResourceVersionTable, error) { for _, udpListener := range ir.UDP { // 1:1 between IR UDPListener and xDS Cluster - xdsCluster, err := buildXdsCluster(udpListener.Name, udpListener.Destinations) + xdsCluster, err := buildXdsCluster(udpListener.Name, udpListener.Destinations, false /*isHTTP2 */) if err != nil { return nil, multierror.Append(err, errors.New("error building xds cluster")) } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 6ed5dd282c9..5591363f815 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -65,6 +65,9 @@ func TestTranslate(t *testing.T) { { name: "udp-route", }, + { + name: "http2-route", + }, } for _, tc := range testCases { From c1f2585c2de5eca380d21b93a1c3815b1013d65d Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Fri, 11 Nov 2022 00:37:47 -0800 Subject: [PATCH 105/113] Fixes Envoy Deployment Ref in Dev Doc (#722) --- docs/latest/dev/README.md | 2 +- docs/v0.2.0/dev/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/latest/dev/README.md b/docs/latest/dev/README.md index e64c2718ced..f66661406d3 100644 --- a/docs/latest/dev/README.md +++ b/docs/latest/dev/README.md @@ -119,7 +119,7 @@ export ENVOY_DEPLOYMENT=$(kubectl get deploy -n envoy-gateway-system --selector= Port forward the admin interface port: ```shell -kubectl port-forward deploy/envoy-${ENVOY_DEPLOYMENT} -n envoy-gateway-system 19000:19000 +kubectl port-forward deploy/${ENVOY_DEPLOYMENT} -n envoy-gateway-system 19000:19000 ``` Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. diff --git a/docs/v0.2.0/dev/README.md b/docs/v0.2.0/dev/README.md index e64c2718ced..f66661406d3 100644 --- a/docs/v0.2.0/dev/README.md +++ b/docs/v0.2.0/dev/README.md @@ -119,7 +119,7 @@ export ENVOY_DEPLOYMENT=$(kubectl get deploy -n envoy-gateway-system --selector= Port forward the admin interface port: ```shell -kubectl port-forward deploy/envoy-${ENVOY_DEPLOYMENT} -n envoy-gateway-system 19000:19000 +kubectl port-forward deploy/${ENVOY_DEPLOYMENT} -n envoy-gateway-system 19000:19000 ``` Now you are able to view the running Envoy configuration by navigating to `127.0.0.1:19000/config_dump`. From 36e65c1ca2fd0c53d5c8cc30d8088689cd1e4d6b Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sun, 9 Oct 2022 08:34:35 +0000 Subject: [PATCH 106/113] Add authn policy design with JWT only Signed-off-by: Lizan Zhou --- docs/latest/design/authentication_policy.md | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/latest/design/authentication_policy.md diff --git a/docs/latest/design/authentication_policy.md b/docs/latest/design/authentication_policy.md new file mode 100644 index 00000000000..db2d87336e4 --- /dev/null +++ b/docs/latest/design/authentication_policy.md @@ -0,0 +1,40 @@ +# Envoy Gateway Authentication Policy API + +## Overview + +This authen policy is to declare the authentication mechanisms, to be enforce on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) +The policy is similar to [OpenAPI 3.1 security objects](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#securitySchemeObject) without the API key part, and should be easily translatable from it with some additions. + +## Authentication mechanisms +This policy should support the following authentication mechanisms: +- JWT Bearer Token +- mutualTLS (client certificate) +- OAuth2 +- OIDC +- External authentication + +In the first phase, Envoy Gateway will implement JWT Bearer Token authentication. + +In general those policy translates into Envoy HTTP filters at HTTP connection manager level, and route specific settings will be applied for each route. These APIs are expressed in a Policy CRD and attached to Gateway API resources with [Policy Attachement](https://gateway-api.sigs.k8s.io/references/policy-attachment/). + +## JWT Bearer Token + +A JWT Bearer Token authentication policy will look like the following: + +``` +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Authentication +metadata: + name: productpage +spec: + type: jwt + jwt: + iss: https://www.okta.com + aud: bookinfo.com + jwksUri: https://bookinfo.com/jwt/public-key/jwks.json + targetRef: + kind: HTTPRoute + name: httpbin +``` + +JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refersh. From 12284786d871545723b45ee2797722b2e0612275 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 2 Nov 2022 10:45:07 -0700 Subject: [PATCH 107/113] Update docs/design/authentication_policy.md Co-authored-by: Arko Dasgupta Signed-off-by: Lizan Zhou --- docs/latest/design/authentication_policy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/latest/design/authentication_policy.md b/docs/latest/design/authentication_policy.md index db2d87336e4..65d8c72bfcf 100644 --- a/docs/latest/design/authentication_policy.md +++ b/docs/latest/design/authentication_policy.md @@ -2,7 +2,7 @@ ## Overview -This authen policy is to declare the authentication mechanisms, to be enforce on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) +This authentication policy declares the authentication mechanisms, to be enforced on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) The policy is similar to [OpenAPI 3.1 security objects](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#securitySchemeObject) without the API key part, and should be easily translatable from it with some additions. ## Authentication mechanisms From ba5b9b69e2208e2774f5f59b5608299678e9d7ba Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 2 Nov 2022 10:45:13 -0700 Subject: [PATCH 108/113] Update docs/design/authentication_policy.md Co-authored-by: Arko Dasgupta Signed-off-by: Lizan Zhou --- docs/latest/design/authentication_policy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/latest/design/authentication_policy.md b/docs/latest/design/authentication_policy.md index 65d8c72bfcf..5a8d072b9ed 100644 --- a/docs/latest/design/authentication_policy.md +++ b/docs/latest/design/authentication_policy.md @@ -37,4 +37,4 @@ spec: name: httpbin ``` -JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refersh. +JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refresh. From 69923b3ec364bf4841b92e878b7f89db313a6b01 Mon Sep 17 00:00:00 2001 From: danehans Date: Thu, 3 Nov 2022 08:27:01 -0700 Subject: [PATCH 109/113] Refactors request auth design doc Signed-off-by: danehans --- docs/latest/design/request-authentication.md | 384 +++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 docs/latest/design/request-authentication.md diff --git a/docs/latest/design/request-authentication.md b/docs/latest/design/request-authentication.md new file mode 100644 index 00000000000..5acb9398e72 --- /dev/null +++ b/docs/latest/design/request-authentication.md @@ -0,0 +1,384 @@ +# Request Authentication + +## Overview + +[Issue 336][] specifies the need for exposing a user-facing API to configure request authentication. Request +authentication is defined as an authentication mechanism to be enforced by Envoy on a per-request basis. A connection +will be rejected if it contains invalid authentication information, based on the `Authentication` API type proposed in +this design document. + +Envoy Gateway leverages [Gateway API][] for configuring managed Envoy proxies. Gateway API defines core, extended, and +implementation-specific API [support levels][] for implementors such as Envoy Gateway to expose features. Since +implementing request authentication is not covered by `Core` or `Extended` APIs, an `Implementation-specific` API will +be created for this purpose. + +## Goals + +* Define an API for configuring request authentication. +* Implement [JWT] as the first supported authentication type. +* Allow users that manage routes, e.g. [HTTPRoute][], to authenticate matching requests before forwarding to a backend + service. +* Support HTTPRoutes as an Authentication API referent. + +## Non-Goals + +* Allow infrastructure administrators to override or establish default authentication policies. +* Support referents other than HTTPRoute. + +## Use-Cases + +These use-cases are presented as an aid for how users may attempt to utilize the outputs of the design. They are not an +exhaustive list of features for authentication support in Envoy Gateway. + +As a Service Producer, I need the ability to: +* Authenticate a request before forwarding it to a backend service. +* Have different authentication mechanisms per route rule. +* Choose from different authentication mechanisms supported by Envoy, e.g. OIDC. + +### Security API Group + +A new API group, `security.gateway.envoyproxy.io` is introduced to group security-related APIs. This will allow security +APIs to be versioned, changed, etc. over time. + +### Authentication API Type + +The Authentication API type defines authentication configuration for authenticating requests through managed Envoy +proxies. + +```go +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +) + +type Authentication struct { + metav1.TypeMeta + metav1.ObjectMeta + + // Spec defines the desired state of the Authentication type. + Spec AuthenticationSpec + + // Note: The status sub-resource has been excluded but may be added in the future. +} + +// AuthenticationSpec defines the desired state of the Authentication type. +// +union +type AuthenticationSpec struct { + // Type defines the type of authentication provider to use. Supported provider + // types are: + // + // * JWT + // + // JWT defines the JSON Web Token (JWT) authentication provider type. + // + // +unionDiscriminator + Type AuthenticationType `json:"type"` + + // JWT defines the JSON Web Token (JWT) authentication provider type. When multiple + // jwtProviders are specified, the JWT is considered valid if any of the providers + // successfully validate the JWT. For additional details, see: + // + // https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/jwt_authn_filter.html + // + // +kubebuilder:validation:MaxItems=4 + // +optional + JwtProviders []JwtAuthenticationProvider `json:"jwtProviders,omitempty"` +} + +// AuthenticationType is a type of authentication provider. +// +kubebuilder:validation:Enum=JWT +type AuthenticationType string + +const ( + // JwtAuthenticationProviderType is the JWT authentication provider type. + JwtAuthenticationProviderType AuthenticationType = "JWT" +) + +// JwtAuthenticationProvider defines the JSON Web Token (JWT) authentication provider type +// and how JWTs should be verified: +type JwtAuthenticationProvider struct { + // Name defines a unique name for the JWT provider. A name can have a variety of forms, + // including RFC1123 subdomains, RFC 1123 labels, or RFC 1035 labels. + // + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` + + // Issuer is the principal that issued the JWT. For additional details, see: + // + // https://tools.ietf.org/html/rfc7519#section-4.1.1 + // + // Example: + // issuer: https://auth.example.com + // + // If not provided, the JWT issuer is not checked. + // + // +kubebuilder:validation:MaxLength=253 + // +optional + Issuer string `json:"issuer,omitempty"` + + // Audiences is a list of JWT audiences allowed to access. For additional details, see: + // + // https://tools.ietf.org/html/rfc7519#section-4.1.3 + // + // Example: + // audiences: + // - foo.apps.example.com + // bar.apps.example.com + // + // If not provided, JWT audiences are not checked. + // + // +kubebuilder:validation:MaxItems=8 + // +optional + Audiences []string `json:"audiences,omitempty"` + + // RemoteJWKS defines how to fetch and cache JSON Web Key Sets (JWKS) from a remote + // HTTP/HTTPS endpoint. + RemoteJWKS RemoteJWKS `json:"remoteJWKS"` + + // TODO: Add TBD JWT fields based on defined use cases. +} + +// RemoteJWKS defines how to fetch and cache JSON Web Key Sets (JWKS) from a remote +// HTTP/HTTPS endpoint. +type RemoteJWKS struct { + // Uri is the HTTP/HTTPS URI to fetch the JWKS. + // + // Example: + // uri: https://www.googleapis.com/oauth2/v1/certs + // + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Uri string `json:"uri"` + + // TODO: Add TBD remote JWKS fields based on defined use cases. +} +``` + +The status subresource is not included in the Authentication API. Status will be surfaced by an HTTPRoute that +references an Authentication. For example, an HTTPRoute will surface the `ResolvedRefs=False` status condition if it +references an Authentication that does not exist. It may be beneficial to add status fields in the future based on +defined use-cases. For example, a remote JWKS can be validated based on the specified URI and have an appropriate +status condition surfaced. + +The following is an example of a JWT authentication provider: + +```yaml +apiVersion: security.gateway.envoyproxy.io/v1alpha1 +kind: Authentication +metadata: + name: example-jwt +spec: + type: JWT + jwtProviders: + - name: example + issuer: https://www.example.com + audiences: + - foo.com + remoteJwks: + uri: https://foo.com/jwt/public-key/jwks.json + +status: + +``` + +The following is an example HTTPRoute configured to use the above JWT authentication provider: + +```yaml +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: example-hwt-authn +spec: + parentRefs: + - name: eg + hostnames: + - www.example.com + rules: + - matches: + - path: + type: PathPrefix + value: /foo + filters: + - type: ExtensionRef + extensionRef: + group: security.gateway.envoyproxy.io + kind: Authentication + name: example-jwt + backendRefs: + - name: httpbin + port: 80 +``` + +Requests for `www.example.com/foo` will be authenticated using the referenced JWT provider before being forwarded to the +`httpbin` backend service. + +## Implementation Details + +The JWT authentication type is translated to an Envoy [JWT authentication filter][] and a cluster is created for each +remote JWKS. The following cluster is created for the JWT provider defines in the above Authentication: + +```yaml +clusters: + - name: foo.com|443 + load_assignment: + cluster_name: foo.com|443 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: foo.com + port_value: 443 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: foo.com + common_tls_context: + validation_context: + match_subject_alt_names: + - exact: "*.foo.com" + trusted_ca: + filename: /etc/ssl/certs/ca-certificates.crt +``` + +A JWT authentication HTTP filter is added to the HTTP Connection Manager. For example: + +```yaml +dynamic_resources: + dynamic_listeners: + - name: example_listener + address: + socket_address: + address: 1.2.3.4 + port_value: 80 + filter_chains: + - filters: + - name: envoy.http_connection_manager + http_filters: + - name: envoy.filters.http.jwt_authn + typed_config: + "@type": type.googleapis.com/envoy.config.filter.http.jwt_authn.v2alpha.JwtAuthentication +``` + +This JWT authentication filter contains two fields: +* The `providers` field specifies how a JWT should be verified, such as where to extract the token, where to fetch the + public key (JWKS) and where to output its payload. This field is built from `jwtProviders` of an Authentication. +* The `rules` field specifies matching rules and their requirements. If a request matches a rule, its requirement + applies. The requirement specifies which JWT providers should be used. This field is built from a HTTPRoute + `matches` rule that references the Authentication extended filter. When a referenced Authentication specifies + multiple `jwtProviders`, the JWT is considered valid if __any__ of the providers successfully validate the JWT. + +The following JWT Authentication filter `providers` configuration is created from the above Authentication. + +```yaml +providers: + example: + issuer: https://www.example.com + audiences: + - foo.com + remote_jwks: + http_uri: + uri: https://foo.com/jwt/public-key/jwks.json + cluster: example_jwks_cluster + timeout: 1s +``` + +The following JWT Authentication filter `rules` configuration is created from the above HTTPRoute. + +```yaml +rules: + - match: + prefix: /foo + requires: + provider_name: example +``` + +If the Authentication included a second JWT provider named `example`, the resulting JWT Authentication filter `rules` +would be created: + +```yaml +rules: +- match: + prefix: /foo + requires: + requires_any: + requirements: + - provider_name: example + - provider_name: example2 +``` + +### Implementation Outline + +* Create a `security` API group and add the Authentication API type to this group. +* Update the Kubernetes provider to get/watch Authentication resources that are referenced by managed HTTPRoutes. Add + the referenced Authentication object to the resource map and publish it. +* Update the resource translator to include the Authentication API in HTTPRoute processing. +* Update the xDS translator to translate an Authentication into xDS resources. The translator should perform the + following: + * Convert a list of JWT rules from the xds IR into an Envoy JWT filter config. + * Create a JWT authentication filter. + * Build the HTTP Connection Manager (MCM) HTTP filters. + * Build the HCM. + * When building the Listener, create an HCM for each filter-chain. + +## Adding Authentication Provider Types + +Additional authentication provider types can be added in the future through the `ProviderType` API. For example, to add +the `Foo` authentication provider type: + +Define the `FooProvider` type: + +```go +package v1alpha1 + +// FooProvider defines the "Foo" authentication provider type. +type FooProvider struct { + // TODO: Define fields of the Foo authentication provider type. +} +``` + +Add the `FooProvider` type to `AuthenticationSpec`: + +```go +package v1alpha1 + +type AuthenticationSpec struct { + ... + + // Foo defines the Foo authentication type. For additional + // details, see: + // + // + // + // +optional + Foo *FooAuthentication `json:"foo"` +} +``` + +Authentication should support additional authentication types in the future, for example: +- mutualTLS (client certificate) +- OAuth2 +- OIDC +- External authentication + +## Outstanding Questions + +- If Envoy Gateway owns the Authentication API, is an xDS IR equivalent needed? +- Should local JWKS be implemented before remote JWKS? +- How should Envoy obtain the trusted CA for a remote JWKS? +- Should HTTPS be the only supported scheme for remote JWKS? +- Should OR'ing JWT providers be supported? +- Should Authentication provide status? +- Are the API field validation rules acceptable? + +[Issue 336]: https://github.com/envoyproxy/gateway/issues/336 +[Gateway API]: https://gateway-api.sigs.k8s.io/ +[support levels]: https://gateway-api.sigs.k8s.io/concepts/conformance/?h=extended#2-support-levels +[JWT]: https://jwt.io/ +[HTTPRoute]: https://gateway-api.sigs.k8s.io/api-types/httproute/ +[JWKS]: https://www.rfc-editor.org/rfc/rfc7517 +[JWT authentication filter]: https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/jwt_authn_filter#config-http-filters-jwt-authn From d8211237725e52f338e045c0b447e5e405845195 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sun, 9 Oct 2022 08:34:35 +0000 Subject: [PATCH 110/113] Add authn policy design with JWT only Signed-off-by: Lizan Zhou --- docs/latest/design/authentication_policy.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/latest/design/authentication_policy.md b/docs/latest/design/authentication_policy.md index 5a8d072b9ed..d632afa2356 100644 --- a/docs/latest/design/authentication_policy.md +++ b/docs/latest/design/authentication_policy.md @@ -2,7 +2,11 @@ ## Overview +<<<<<<< HEAD:docs/latest/design/authentication_policy.md This authentication policy declares the authentication mechanisms, to be enforced on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) +======= +This authen policy is to declare the authentication mechanisms, to be enforce on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) +>>>>>>> 9a6ed41 (Add authn policy design with JWT only):docs/design/authentication_policy.md The policy is similar to [OpenAPI 3.1 security objects](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#securitySchemeObject) without the API key part, and should be easily translatable from it with some additions. ## Authentication mechanisms @@ -37,4 +41,8 @@ spec: name: httpbin ``` +<<<<<<< HEAD:docs/latest/design/authentication_policy.md JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refresh. +======= +JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refersh. +>>>>>>> 9a6ed41 (Add authn policy design with JWT only):docs/design/authentication_policy.md From 33772f72f7dda27e471c2beffc6ec6579a4f7d83 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 2 Nov 2022 10:45:07 -0700 Subject: [PATCH 111/113] Update docs/design/authentication_policy.md Co-authored-by: Arko Dasgupta Signed-off-by: Lizan Zhou --- docs/latest/design/authentication_policy.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/latest/design/authentication_policy.md b/docs/latest/design/authentication_policy.md index d632afa2356..46b7f169d01 100644 --- a/docs/latest/design/authentication_policy.md +++ b/docs/latest/design/authentication_policy.md @@ -2,11 +2,15 @@ ## Overview +<<<<<<< HEAD:docs/latest/design/authentication_policy.md <<<<<<< HEAD:docs/latest/design/authentication_policy.md This authentication policy declares the authentication mechanisms, to be enforced on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) ======= This authen policy is to declare the authentication mechanisms, to be enforce on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) >>>>>>> 9a6ed41 (Add authn policy design with JWT only):docs/design/authentication_policy.md +======= +This authentication policy declares the authentication mechanisms, to be enforced on connection and request going though Envoy Gateway. This includes the credential (X.509, JWT, etc), parameters (cipher suites, key algorithms) +>>>>>>> b5c4755 (Update docs/design/authentication_policy.md):docs/design/authentication_policy.md The policy is similar to [OpenAPI 3.1 security objects](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#securitySchemeObject) without the API key part, and should be easily translatable from it with some additions. ## Authentication mechanisms From aa4c617b3c0be043b825c85ca5342a9c472d71d5 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 2 Nov 2022 10:45:13 -0700 Subject: [PATCH 112/113] Update docs/design/authentication_policy.md Co-authored-by: Arko Dasgupta Signed-off-by: Lizan Zhou --- docs/latest/design/authentication_policy.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/latest/design/authentication_policy.md b/docs/latest/design/authentication_policy.md index 46b7f169d01..761d93bccc1 100644 --- a/docs/latest/design/authentication_policy.md +++ b/docs/latest/design/authentication_policy.md @@ -45,8 +45,12 @@ spec: name: httpbin ``` +<<<<<<< HEAD:docs/latest/design/authentication_policy.md <<<<<<< HEAD:docs/latest/design/authentication_policy.md JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refresh. ======= JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refersh. >>>>>>> 9a6ed41 (Add authn policy design with JWT only):docs/design/authentication_policy.md +======= +JWT Bearer token will be translate to Envoy's JWT authentication filter. The JWKS URI need to be translated to a separate cluster for JWKS fetch and refresh. +>>>>>>> 3cb97e1 (Update docs/design/authentication_policy.md):docs/design/authentication_policy.md From 12f33db7a8fa599935d0ef5b10f7f9054fb2f289 Mon Sep 17 00:00:00 2001 From: danehans Date: Fri, 11 Nov 2022 09:07:56 -0800 Subject: [PATCH 113/113] Resolves Review Feedback Signed-off-by: danehans --- docs/latest/design/request-authentication.md | 272 +++++++++++++++++-- 1 file changed, 249 insertions(+), 23 deletions(-) diff --git a/docs/latest/design/request-authentication.md b/docs/latest/design/request-authentication.md index 5acb9398e72..be7b8bd3f53 100644 --- a/docs/latest/design/request-authentication.md +++ b/docs/latest/design/request-authentication.md @@ -18,12 +18,14 @@ be created for this purpose. * Implement [JWT] as the first supported authentication type. * Allow users that manage routes, e.g. [HTTPRoute][], to authenticate matching requests before forwarding to a backend service. -* Support HTTPRoutes as an Authentication API referent. +* Support HTTPRoutes as an Authentication API referent. HTTPRoute provides multiple [extension points][]. The + [HTTPRouteFilter][] is the extension point supported by the Authentication API. ## Non-Goals * Allow infrastructure administrators to override or establish default authentication policies. * Support referents other than HTTPRoute. +* Support Gateway API extension points other than HTTPRouteFilter. ## Use-Cases @@ -163,13 +165,15 @@ references an Authentication that does not exist. It may be beneficial to add st defined use-cases. For example, a remote JWKS can be validated based on the specified URI and have an appropriate status condition surfaced. -The following is an example of a JWT authentication provider: +#### Authentication Example + +The following is an Authentication example with one JWT authentication provider: ```yaml apiVersion: security.gateway.envoyproxy.io/v1alpha1 kind: Authentication metadata: - name: example-jwt + name: example spec: type: JWT jwtProviders: @@ -184,13 +188,16 @@ status: ``` +__Note:__ `type` is a union type, allowing only one of any supported provider type such as `jwtProviders` to be +specified. + The following is an example HTTPRoute configured to use the above JWT authentication provider: ```yaml apiVersion: gateway.networking.k8s.io/v1beta1 kind: HTTPRoute metadata: - name: example-hwt-authn + name: example spec: parentRefs: - name: eg @@ -206,22 +213,27 @@ spec: extensionRef: group: security.gateway.envoyproxy.io kind: Authentication - name: example-jwt + name: example backendRefs: - - name: httpbin - port: 80 + - name: backend + port: 3000 ``` Requests for `www.example.com/foo` will be authenticated using the referenced JWT provider before being forwarded to the -`httpbin` backend service. +backend service named "backend". ## Implementation Details The JWT authentication type is translated to an Envoy [JWT authentication filter][] and a cluster is created for each -remote JWKS. The following cluster is created for the JWT provider defines in the above Authentication: +remote JWKS. The following examples provide additional details on how Gateway API and Authentication resources are +translated into Envoy configuration. + +### Example 1: One Route with One JWT Provider + +The following cluster is created from the above HTTPRoute and Authentication: ```yaml -clusters: +dynamic_clusters: - name: foo.com|443 load_assignment: cluster_name: foo.com|443 @@ -266,7 +278,8 @@ dynamic_resources: This JWT authentication filter contains two fields: * The `providers` field specifies how a JWT should be verified, such as where to extract the token, where to fetch the - public key (JWKS) and where to output its payload. This field is built from `jwtProviders` of an Authentication. + public key (JWKS) and where to output its payload. This field is built from the source resource `namespace-name`, and + the JWT provider name of an Authentication. * The `rules` field specifies matching rules and their requirements. If a request matches a rule, its requirement applies. The requirement specifies which JWT providers should be used. This field is built from a HTTPRoute `matches` rule that references the Authentication extended filter. When a referenced Authentication specifies @@ -294,23 +307,234 @@ rules: - match: prefix: /foo requires: - provider_name: example + provider_name: default-example-example ``` -If the Authentication included a second JWT provider named `example`, the resulting JWT Authentication filter `rules` -would be created: +### Example 2: Two HTTPRoutes with Different Authentications + +The following example contains: +* Two HTTPRoutes with different hostnames. +* Each HTTPRoute references a different Authentication +* Each Authentication contains a different JWT provider. ```yaml -rules: -- match: - prefix: /foo - requires: - requires_any: - requirements: - - provider_name: example - - provider_name: example2 +apiVersion: security.gateway.envoyproxy.io/v1alpha1 +kind: Authentication +metadata: + name: example1 +spec: + type: JWT + jwtProviders: + - name: example1 + issuer: https://www.example1.com + audiences: + - foo.com + remoteJwks: + uri: https://foo.com/jwt/public-key/jwks.json +--- +apiVersion: security.gateway.envoyproxy.io/v1alpha1 +kind: Authentication +metadata: + name: example2 +spec: + type: JWT + jwtProviders: + - name: example2 + issuer: https://www.example2.com + audiences: + - bar.com + remoteJwks: + uri: https://bar.com/jwt/public-key/jwks.json +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: example1 +spec: + hostnames: + - www.example1.com + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: eg + rules: + - matches: + - path: + type: PathPrefix + value: /foo + filters: + - type: ExtensionRef + extensionRef: + group: security.gateway.envoyproxy.io + kind: Authentication + name: example1 + backendRefs: + - name: backend + port: 3000 +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: example2 +spec: + hostnames: + - www.example2.com + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: eg + rules: + - matches: + - path: + type: PathPrefix + value: /bar + filters: + - type: ExtensionRef + extensionRef: + group: security.gateway.envoyproxy.io + kind: Authentication + name: example2 + backendRefs: + - name: backend2 + port: 3000 +``` + +The following xDS configuration is created from the above example resources: + +```yaml +configs: +... +dynamic_listeners: + - name: default-eg-http + ... + default_filter_chain: + filters: + - name: envoy.filters.network.http_connection_manager_1 + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: http + rds: + config_source: + ... + route_config_name: default-eg-http-1 + http_filters: + - name: envoy.filters.http.jwt_authn + typed_config: + '@type': >- + type.googleapis.com/envoy.config.filter.http.jwt_authn.v2alpha.JwtAuthentication + providers: + default-example1-example1: + issuer: https://www.example1.com + audiences: + - foo.com + remote_jwks: + http_uri: + uri: https://foo.com/jwt/public-key/jwks.json + cluster: default-example1-example1-jwt + rules: + - match: + exact: /foo + requires: + provider_name: default-example1-example1 + - name: envoy.filters.http.router + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + - name: envoy.filters.network.http_connection_manager_2 + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: http + rds: + config_source: + ... + route_config_name: default-eg-http-2 + http_filters: + - name: envoy.filters.http.jwt_authn + typed_config: + '@type': >- + type.googleapis.com/envoy.config.filter.http.jwt_authn.v2alpha.JwtAuthentication + providers: + default-example2-example2: + issuer: https://www.example2.com + audiences: + - bar.com + remote_jwks: + http_uri: + uri: https://bar.com/jwt/public-key/jwks.json + cluster: default-example2-example2-jwt + rules: + - match: + exact: /bar + requires: + provider_name: default-example2-example2 + - name: envoy.filters.http.router + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router +dynamic_route_configs: + - route_config: + '@type': type.googleapis.com/envoy.config.route.v3.RouteConfiguration + name: default-eg-http-1 + virtual_hosts: + - name: default-eg-http-1 + domains: + - '*' + routes: + - match: + prefix: /foo + headers: + - name: ':authority' + string_match: + exact: www.example1.com + route: + cluster: default-backend-rule-0-match-0-www.example1.com + - route_config: + '@type': type.googleapis.com/envoy.config.route.v3.RouteConfiguration + name: default-eg-http + virtual_hosts: + - name: default-eg-http-2 + domains: + - '*' + routes: + - match: + prefix: /bar + headers: + - name: ':authority' + string_match: + exact: www.example2.com + route: + cluster: default-backend2-rule-0-match-0-www.example2.com +dynamic_active_clusters: + - cluster: + name: default-backend-rule-0-match-0-www.example.com + ... + endpoints: + - locality: {} + lb_endpoints: + - endpoint: + address: + socket_address: + address: $BACKEND_SERVICE1_IP + port_value: 3000 + - cluster: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: default-backend-rule-1-match-0-www.example.com + ... + endpoints: + - locality: {} + lb_endpoints: + - endpoint: + address: + socket_address: + address: $BACKEND_SERVICE2_IP + port_value: 3000 +... ``` +__Note:__ The JWT provider cluster and route is omitted from the above example for brevity. + ### Implementation Outline * Create a `security` API group and add the Authentication API type to this group. @@ -321,7 +545,7 @@ rules: following: * Convert a list of JWT rules from the xds IR into an Envoy JWT filter config. * Create a JWT authentication filter. - * Build the HTTP Connection Manager (MCM) HTTP filters. + * Build the HTTP Connection Manager (HCM) HTTP filters. * Build the HCM. * When building the Listener, create an HCM for each filter-chain. @@ -380,5 +604,7 @@ Authentication should support additional authentication types in the future, for [support levels]: https://gateway-api.sigs.k8s.io/concepts/conformance/?h=extended#2-support-levels [JWT]: https://jwt.io/ [HTTPRoute]: https://gateway-api.sigs.k8s.io/api-types/httproute/ +[extension points]: https://gateway-api.sigs.k8s.io/concepts/api-overview/?h=extension#extension-points +[HTTPRouteFilter]: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteFilter [JWKS]: https://www.rfc-editor.org/rfc/rfc7517 [JWT authentication filter]: https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/jwt_authn_filter#config-http-filters-jwt-authn