From 4fbe557674411cd79d4e058aab8ae29e8892b0f8 Mon Sep 17 00:00:00 2001 From: Kobi Levi Date: Fri, 28 Jul 2023 09:24:31 +0300 Subject: [PATCH 1/3] Check ResolvedRefs condition in Gateway conformance tests --- conformance/tests/gateway-modify-listeners.go | 51 ++++++++++---- ...secret-reference-grant-all-in-namespace.go | 5 ++ ...gateway-secret-reference-grant-specific.go | 5 ++ .../tests/gateway-with-attached-routes.go | 68 +++++++++++++------ 4 files changed, 94 insertions(+), 35 deletions(-) diff --git a/conformance/tests/gateway-modify-listeners.go b/conformance/tests/gateway-modify-listeners.go index fffed977fd..777c77986b 100644 --- a/conformance/tests/gateway-modify-listeners.go +++ b/conformance/tests/gateway-modify-listeners.go @@ -87,11 +87,18 @@ var GatewayModifyListeners = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 1, }, { @@ -100,11 +107,18 @@ var GatewayModifyListeners = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 1, }, } @@ -161,11 +175,18 @@ var GatewayModifyListeners = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 1, }, } diff --git a/conformance/tests/gateway-secret-reference-grant-all-in-namespace.go b/conformance/tests/gateway-secret-reference-grant-all-in-namespace.go index 1fb5fed88c..f9ed96e610 100644 --- a/conformance/tests/gateway-secret-reference-grant-all-in-namespace.go +++ b/conformance/tests/gateway-secret-reference-grant-all-in-namespace.go @@ -55,6 +55,11 @@ var GatewaySecretReferenceGrantAllInNamespace = suite.ConformanceTest{ Status: metav1.ConditionTrue, Reason: string(v1beta1.ListenerReasonProgrammed), }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, }, AttachedRoutes: 0, }} diff --git a/conformance/tests/gateway-secret-reference-grant-specific.go b/conformance/tests/gateway-secret-reference-grant-specific.go index 6d9c092fc5..ae883d3092 100644 --- a/conformance/tests/gateway-secret-reference-grant-specific.go +++ b/conformance/tests/gateway-secret-reference-grant-specific.go @@ -55,6 +55,11 @@ var GatewaySecretReferenceGrantSpecific = suite.ConformanceTest{ Status: metav1.ConditionTrue, Reason: string(v1beta1.ListenerReasonProgrammed), }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, }, AttachedRoutes: 0, }} diff --git a/conformance/tests/gateway-with-attached-routes.go b/conformance/tests/gateway-with-attached-routes.go index b33040add7..53b5659c23 100644 --- a/conformance/tests/gateway-with-attached-routes.go +++ b/conformance/tests/gateway-with-attached-routes.go @@ -47,11 +47,18 @@ var GatewayWithAttachedRoutes = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 1, }} @@ -66,11 +73,18 @@ var GatewayWithAttachedRoutes = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 2, }} @@ -97,11 +111,18 @@ var GatewayWithAttachedRoutesWithPort8080 = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 0, }, { @@ -110,11 +131,18 @@ var GatewayWithAttachedRoutesWithPort8080 = suite.ConformanceTest{ Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), Kind: v1beta1.Kind("HTTPRoute"), }}, - Conditions: []metav1.Condition{{ - Type: string(v1beta1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }}, + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, + }, AttachedRoutes: 1, }, } From ac5e4ea1bfd943a44073295b95a9add266d22074 Mon Sep 17 00:00:00 2001 From: Kobi Levi Date: Thu, 3 Aug 2023 13:38:04 +0300 Subject: [PATCH 2/3] Check resolvedRefs is enforced at GatewayAndHTTPRoutesMustBeAccepted --- conformance/utils/kubernetes/helpers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go index aec66fa3b9..c82de86db7 100644 --- a/conformance/utils/kubernetes/helpers.go +++ b/conformance/utils/kubernetes/helpers.go @@ -303,6 +303,11 @@ func GatewayAndHTTPRoutesMustBeAccepted(t *testing.T, c client.Client, timeoutCo Status: metav1.ConditionTrue, Reason: string(v1beta1.RouteReasonAccepted), }, + { + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + }, }, }) } From 6ced93580432dd4c53f9f73101f01c79208a60e3 Mon Sep 17 00:00:00 2001 From: Kobi Levi Date: Sun, 17 Sep 2023 19:13:51 +0300 Subject: [PATCH 3/3] Add GatewayListenersMustHaveCondition helper function --- conformance/utils/config/timeout.go | 40 ++++++++++------- conformance/utils/kubernetes/helpers.go | 57 ++++++++++++++++++------- 2 files changed, 66 insertions(+), 31 deletions(-) diff --git a/conformance/utils/config/timeout.go b/conformance/utils/config/timeout.go index dbaf2bec0f..4390063581 100644 --- a/conformance/utils/config/timeout.go +++ b/conformance/utils/config/timeout.go @@ -39,6 +39,10 @@ type TimeoutConfig struct { // Max value for conformant implementation: None GatewayStatusMustHaveListeners time.Duration + // GatewayListenersMustHaveCondition represents the maximum time for a Gateway to have all listeners with a specific condition. + // Max value for conformant implementation: None + GatewayListenersMustHaveCondition time.Duration + // GWCMustBeAccepted represents the maximum time for a GatewayClass to have an Accepted condition set to true. // Max value for conformant implementation: None GWCMustBeAccepted time.Duration @@ -91,22 +95,23 @@ type TimeoutConfig struct { // DefaultTimeoutConfig populates a TimeoutConfig with the default values. func DefaultTimeoutConfig() TimeoutConfig { return TimeoutConfig{ - CreateTimeout: 60 * time.Second, - DeleteTimeout: 10 * time.Second, - GetTimeout: 10 * time.Second, - GatewayMustHaveAddress: 180 * time.Second, - GatewayStatusMustHaveListeners: 60 * time.Second, - GWCMustBeAccepted: 180 * time.Second, - HTTPRouteMustNotHaveParents: 60 * time.Second, - HTTPRouteMustHaveCondition: 60 * time.Second, - TLSRouteMustHaveCondition: 60 * time.Second, - RouteMustHaveParents: 60 * time.Second, - ManifestFetchTimeout: 10 * time.Second, - MaxTimeToConsistency: 30 * time.Second, - NamespacesMustBeReady: 300 * time.Second, - RequestTimeout: 10 * time.Second, - LatestObservedGenerationSet: 60 * time.Second, - RequiredConsecutiveSuccesses: 3, + CreateTimeout: 60 * time.Second, + DeleteTimeout: 10 * time.Second, + GetTimeout: 10 * time.Second, + GatewayMustHaveAddress: 180 * time.Second, + GatewayStatusMustHaveListeners: 60 * time.Second, + GatewayListenersMustHaveCondition: 60 * time.Second, + GWCMustBeAccepted: 180 * time.Second, + HTTPRouteMustNotHaveParents: 60 * time.Second, + HTTPRouteMustHaveCondition: 60 * time.Second, + TLSRouteMustHaveCondition: 60 * time.Second, + RouteMustHaveParents: 60 * time.Second, + ManifestFetchTimeout: 10 * time.Second, + MaxTimeToConsistency: 30 * time.Second, + NamespacesMustBeReady: 300 * time.Second, + RequestTimeout: 10 * time.Second, + LatestObservedGenerationSet: 60 * time.Second, + RequiredConsecutiveSuccesses: 3, } } @@ -127,6 +132,9 @@ func SetupTimeoutConfig(timeoutConfig *TimeoutConfig) { if timeoutConfig.GatewayStatusMustHaveListeners == 0 { timeoutConfig.GatewayStatusMustHaveListeners = defaultTimeoutConfig.GatewayStatusMustHaveListeners } + if timeoutConfig.GatewayListenersMustHaveCondition == 0 { + timeoutConfig.GatewayListenersMustHaveCondition = defaultTimeoutConfig.GatewayListenersMustHaveCondition + } if timeoutConfig.GWCMustBeAccepted == 0 { timeoutConfig.GWCMustBeAccepted = defaultTimeoutConfig.GWCMustBeAccepted } diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go index c82de86db7..b7cc41f8e2 100644 --- a/conformance/utils/kubernetes/helpers.go +++ b/conformance/utils/kubernetes/helpers.go @@ -267,9 +267,12 @@ func MeshNamespacesMustBeReady(t *testing.T, c client.Client, timeoutConfig conf require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", ")) } -// GatewayAndHTTPRoutesMustBeAccepted waits until the specified Gateway has an IP -// address assigned to it and the Route has a ParentRef referring to the -// Gateway. The test will fail if these conditions are not met before the +// GatewayAndHTTPRoutesMustBeAccepted waits until: +// 1. The specified Gateway has an IP address assigned to it. +// 2. The route has a ParentRef referring to the Gateway. +// 3. All the gateway's listeners have the ListenerConditionResolvedRefs set to true. +// +// The test will fail if these conditions are not met before the // timeouts. func GatewayAndHTTPRoutesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, controllerName string, gw GatewayRef, routeNNs ...types.NamespacedName) string { t.Helper() @@ -297,23 +300,24 @@ func GatewayAndHTTPRoutesMustBeAccepted(t *testing.T, c client.Client, timeoutCo SectionName: listener, }, ControllerName: v1beta1.GatewayController(controllerName), - Conditions: []metav1.Condition{ - { - Type: string(v1beta1.RouteConditionAccepted), - Status: metav1.ConditionTrue, - Reason: string(v1beta1.RouteReasonAccepted), - }, - { - Type: string(v1beta1.ListenerConditionResolvedRefs), - Status: metav1.ConditionTrue, - Reason: "", // any reason - }, - }, + Conditions: []metav1.Condition{{ + Type: string(v1beta1.RouteConditionAccepted), + Status: metav1.ConditionTrue, + Reason: string(v1beta1.RouteReasonAccepted), + }}, }) } HTTPRouteMustHaveParents(t, c, timeoutConfig, routeNN, parents, namespaceRequired) } + resolvedRefsCondition := metav1.Condition{ + Type: string(v1beta1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "", // any reason + } + + GatewayListenersMustHaveCondition(t, c, timeoutConfig, gw.NamespacedName, resolvedRefsCondition) + return gwAddr } @@ -352,6 +356,29 @@ func WaitForGatewayAddress(t *testing.T, client client.Client, timeoutConfig con return net.JoinHostPort(ipAddr, port), waitErr } +// GatewayListenersMustHaveCondition checks if every listener of the specified gateway has a +// certain condition. +func GatewayListenersMustHaveCondition(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, gwName types.NamespacedName, condition metav1.Condition) { + t.Helper() + + waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.GatewayListenersMustHaveCondition, true, func(ctx context.Context) (bool, error) { + var gw v1beta1.Gateway + if err := client.Get(ctx, gwName, &gw); err != nil { + return false, fmt.Errorf("error fetching Gateway: %w", err) + } + + for _, listener := range gw.Status.Listeners { + if !findConditionInList(t, listener.Conditions, condition.Type, string(condition.Status), condition.Reason) { + return false, nil + } + } + + return true, nil + }) + + require.NoErrorf(t, waitErr, "error waiting for Gateway status to have a Condition matching expectations on all listeners") +} + // GatewayMustHaveZeroRoutes validates that the gateway has zero routes attached. The status // may indicate a single listener with zero attached routes or no listeners. func GatewayMustHaveZeroRoutes(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, gwName types.NamespacedName) {