Update service labels in NFR tests (#3386)

sjberman · web-flow · commit ee195fc2e045 · 2025-05-20T21:07:58.000Z
Problem: The GKE service label for nginx LB is not provided in the helm chart anymore.

Solution: Add the label in each Gateway infrastructure field.
diff --git a/tests/Makefile b/tests/Makefile
@@ -9,7 +9,6 @@ GITHUB_OUTPUT =
 GW_API_VERSION ?= $(shell sed -n 's/.*ref=v\(.*\)/\1/p' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF
 GW_API_PREV_VERSION ?= 1.2.1## Supported Gateway API version from previous NGF release
 GW_SERVICE_TYPE = NodePort## Service type to use for the gateway
-GW_SVC_GKE_INTERNAL = false
 NGF_VERSION ?= edge## NGF version to be tested
 PULL_POLICY = Never## Pull policy for the images
 NGINX_CONF_DIR = internal/mode/static/nginx/conf
@@ -127,9 +126,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
-		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --plus-enabled=$(PLUS_ENABLED) \
-		--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT) \
-		--gke-project=$(GKE_PROJECT)
+		--plus-enabled=$(PLUS_ENABLED) --plus-license-file-name=$(PLUS_LICENSE_FILE) \
+		--plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT) --gke-project=$(GKE_PROJECT)
 
 .PHONY: test
 test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
@@ -141,7 +139,7 @@ test: build-crossplane-image ## Runs the functional tests on your kind k8s clust
 		--image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
-		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --cluster-name=$(CLUSTER_NAME) --plus-enabled=$(PLUS_ENABLED) \
+		--cluster-name=$(CLUSTER_NAME) --plus-enabled=$(PLUS_ENABLED) \
 		--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT)
 
 .PHONY: test-with-plus
diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go
@@ -35,7 +35,6 @@ type InstallationConfig struct {
 	ServiceType          string
 	PlusUsageEndpoint    string
 	Plus                 bool
-	IsGKEInternalLB      bool
 	Telemetry            bool
 }
 
@@ -231,11 +230,6 @@ func setImageArgs(cfg InstallationConfig) []string {
 
 	if cfg.ServiceType != "" {
 		args = append(args, formatValueSet("nginx.service.type", cfg.ServiceType)...)
-		if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB {
-			args = append(
-				args,
-				formatValueSet(`service.annotations.networking\.gke\.io\/load-balancer-type`, "Internal")...)
-		}
 	}
 
 	return args
diff --git a/tests/scripts/remote-scripts/run-nfr-tests.sh b/tests/scripts/remote-scripts/run-nfr-tests.sh
@@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then
     GINKGO_LABEL="longevity-teardown"
 fi
 
-cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}"
+cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}"
 
 if [ "${START_LONGEVITY}" == "true" ]; then
     suite/scripts/longevity-wrk.sh
diff --git a/tests/suite/manifests/dp-perf/gateway.yaml b/tests/suite/manifests/dp-perf/gateway.yaml
@@ -4,6 +4,9 @@ metadata:
   name: cafe
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/longevity/gateway.yaml b/tests/suite/manifests/longevity/gateway.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/ngf-upgrade/gateway-updated.yaml b/tests/suite/manifests/ngf-upgrade/gateway-updated.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/ngf-upgrade/gateway.yaml b/tests/suite/manifests/ngf-upgrade/gateway.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/reconfig/gateway.yaml b/tests/suite/manifests/reconfig/gateway.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/matches.yaml b/tests/suite/manifests/scale/matches.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: listener
     port: 80
diff --git a/tests/suite/manifests/scale/upstreams.yaml b/tests/suite/manifests/scale/upstreams.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: listener
     hostname: "*.example.com"
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-1.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-1.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-10.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-10.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-11.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-11.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-12.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-12.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-13.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-13.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-14.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-14.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-15.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-15.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-16.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-16.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-17.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-17.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-18.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-18.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-19.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-19.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-2.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-2.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-20.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-20.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-21.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-21.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-22.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-22.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-23.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-23.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-24.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-24.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-25.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-25.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-3.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-3.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-4.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-4.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-5.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-5.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-6.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-6.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-7.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-7.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-8.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-8.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/manifests/scale/zero-downtime/gateway-9.yaml b/tests/suite/manifests/scale/zero-downtime/gateway-9.yaml
@@ -4,6 +4,9 @@ metadata:
   name: gateway
 spec:
   gatewayClassName: nginx
+  infrastructure:
+    annotations:
+      service.annotations.networking.gke.io/load-balancer-type: Internal
   listeners:
   - name: http
     port: 80
diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go
@@ -60,7 +60,6 @@ var (
 	versionUnderTest         = flag.String("version-under-test", "", "Version of NGF that is being tested")
 	imagePullPolicy          = flag.String("pull-policy", "", "Image pull policy for NGF images")
 	serviceType              = flag.String("service-type", "NodePort", "Type of service fronting NGF to be deployed")
-	isGKEInternalLB          = flag.Bool("is-gke-internal-lb", false, "Is the LB service GKE internal only")
 	plusEnabled              = flag.Bool("plus-enabled", false, "Is NGINX Plus enabled")
 	plusLicenseFileName      = flag.String("plus-license-file-name", "", "File name containing the NGINX Plus JWT")
 	plusUsageEndpoint        = flag.String("plus-usage-endpoint", "", "Endpoint for reporting NGINX Plus usage")
@@ -222,7 +221,6 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo
 		Namespace:         ngfNamespace,
 		ChartPath:         cfg.chartPath,
 		ServiceType:       *serviceType,
-		IsGKEInternalLB:   *isGKEInternalLB,
 		Plus:              *plusEnabled,
 		PlusUsageEndpoint: *plusUsageEndpoint,
 		Telemetry:         cfg.telemetry,
diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go
