Add NFR workflow (nginx#1621)

* Add NFR workflow

* Initial review feedback

* Fix typos, update docs, and remove image builds unless plus

Author: ciarams87

.github/workflows/nfr.yml

@@ -0,0 +1,176 @@
+name: Non Functional Testing
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_label:
+        description: NFR test to run. Choose between performance, upgrade, or all
+        required: true
+        default: all
+        type: choice
+        options: [performance, upgrade, all]
+      version:
+        description: Version of NGF under test
+        required: true
+        default: edge
+      image_tag:
+        description: Tag of the NGF and NGINX Docker images
+        required: true
+        default: edge
+      nginx_plus:
+        description: Run tests with NGINX Plus
+        required: false
+        default: false
+        type: boolean
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.ref_name }}-nfr
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  setup-and-run-tests:
+    name: Setup and Run NFR Tests
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write # needed for opening PR with the results files
+      pull-requests: write # needed for opening PR with the results files
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: Setup Golang Environment
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: stable
+
+    - name: Set GOPATH
+      run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+
+    - name: Docker Buildx
+      if: ${{ inputs.nginx_plus == true }}
+      uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+    - name: NGINX Docker meta
+      id: nginx-meta
+      if: ${{ inputs.nginx_plus == true }}
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+      with:
+        images: |
+          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx-plus
+        tags: |
+          type=raw,value=${{ inputs.image_tag }}
+
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
+      with:
+        token_format: access_token
+        workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
+        service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
+      with:
+        project_id: ${{ secrets.GCP_PROJECT_ID }}
+        install_components: kubectl
+
+    - name: Login to GCR
+      if: ${{ inputs.nginx_plus == true }}
+      run: gcloud auth configure-docker gcr.io -q
+
+    - name: Build NGINX Plus Docker Image
+      if: ${{ inputs.nginx_plus == true }}
+      uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+      with:
+        file: build/Dockerfile.nginxplus
+        tags: ${{ steps.nginx-meta.outputs.tags }}
+        context: "."
+        platforms: linux/amd64
+        provenance: false
+        pull: true
+        push: true
+        build-args: |
+          NJS_DIR=internal/mode/static/nginx/modules/src
+          NGINX_CONF_DIR=internal/mode/static/nginx/conf
+          BUILD_AGENT=gha
+        secrets: |
+          ${{ format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }}
+          ${{ format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }}
+
+    - name: Setup dotenv file
+      working-directory: ./tests/scripts
+      run: |
+        echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
+        echo "TAG=${{ inputs.image_tag }}" >> vars.env
+        echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
+        echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
+        echo "NGINX_PLUS_PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx-plus" >> vars.env
+        echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
+        echo "GKE_CLUSTER_ZONE=us-east1-b" >> vars.env
+        echo "GKE_CLUSTER_REGION=us-east1" >> vars.env
+        echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
+        echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
+        echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
+        echo "IMAGE=projects/debian-cloud/global/images/debian-11-bullseye-v20240213" >> vars.env
+        echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}" >> vars.env
+        echo "NGF_REPO=nginxinc" >> vars.env
+        echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
+        echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
+        echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
+        echo "PLUS_ENABLED=${{ inputs.nginx_plus }}" >> vars.env
+        echo "GINKGO_LABEL=" >> vars.env
+        echo "NGF_VERSION=${{ inputs.version }}" >> vars.env
+
+    - name: Create GKE cluster
+      working-directory: ./tests
+      run:
+        make create-gke-cluster
+
+    - name: Create and setup VM
+      working-directory: ./tests
+      run:
+        make create-and-setup-vm
+
+    - name: Run Tests
+      working-directory: ./tests
+      run: |
+        if ${{ inputs.test_label != 'all' }}; then
+          sed -i '/^GINKGO_LABEL=/s/=.*/="${{ inputs.test_label }}"/' "scripts/vars.env" && make run-tests-on-vm;
+        else
+          make run-tests-on-vm;
+        fi
+
+    - name: Cleanup
+      working-directory: ./tests
+      if: always()
+      run: |
+        bash scripts/cleanup-vm.sh true
+        make delete-gke-cluster
+        rm -rf scripts/vars.env
+
+    - name: Open a PR with the results
+      uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc # v6.0.1
+      with:
+        commit-message: NFR Test Results for NGF version ${{ inputs.version }}
+        author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>
+        branch: tests/nfr-tests-${{ inputs.version }}
+        delete-branch: true
+        title: NFR Test Results for NGF version ${{ inputs.version }}
+        body: |
+          Update with NFR test results for NGF version ${{ inputs.version }}
+          - Auto-generated by the NFR tests workflow run ${{ github.run_id }}
+          - Tests ran using Docker image tag ${{ inputs.image_tag }}
+          - ${{ inputs.test_label }} test(s) ran
+          - NGINX Plus enabled: ${{ inputs.nginx_plus }}
+        labels: |
+          tests
+        assignees: ${{ github.actor }}
+        draft: true

docs/developer/release-process.md

@@ -28,7 +28,9 @@ To create a new release, follow these steps:
    format `Release X.Y.Z`.
 2. Stop merging any new work into the main branch.
 3. Test the main branch for release-readiness. For that, use the `edge` containers, which are built from the main
-   branch, and the [example applications](/examples).
+   branch, and run the following:
+   1. Run the [example applications](/examples) manually and verify their correctness.
+   2. Kick off the [NFR workflow](https://github.com/nginxinc/nginx-gateway-fabric/actions/nfr.yml) in the browser. For `image_tag`, use `edge`, and for  `version`, use the upcoming `X.Y.Z` NGF version. This will run all of the NFR tests which are automated and open a PR with the results files when it is complete. Review this PR and make any necessary changes before merging.
 4. If a problem is found, prepare a fix PR, merge it into the main branch and return to the previous step.
 5. If the supported Gateway API minor version has changed since the last release, test NGINX Gateway Fabric with the previous version of the Gateway API CRDs.
 6. If a compatibility issue is found, add a note to the release notes explaining that the previous version is not supported.

tests/Makefile

@@ -11,6 +11,7 @@ GW_SERVICE_TYPE=NodePort
 GW_SVC_GKE_INTERNAL=false
 GINKGO_LABEL=
 GINKGO_FLAGS=
+NGF_VERSION=
 
 ifneq ($(GINKGO_LABEL),)
     override GINKGO_FLAGS += -ginkgo.label-filter "$(GINKGO_LABEL)"
@@ -48,9 +49,10 @@ load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind clus
 
 test: ## Run the system tests against your default k8s cluster
 	go test -v ./suite $(GINKGO_FLAGS) -args --gateway-api-version=$(GW_API_VERSION) \
-		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --plus-enabled=$(PLUS_ENABLED) \
-		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
-		--k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
+		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
+		--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) \
+		--pull-policy=$(PULL_POLICY) --k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) \
+		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
@@ -80,4 +82,12 @@ cleanup-router: ## Delete the GKE router
 setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the tests
 
 .PHONY: cleanup-gcp
-cleanup-gcp: cleanup-router cleanup-vm ## Cleanup all GCP resources
+cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources
+
+.PHONY: create-gke-cluster
+create-gke-cluster: ## Create a GKE cluster
+	bash scripts/create-gke-cluster.sh
+
+.PHONY: delete-gke-cluster
+delete-gke-cluster: ## Delete the GKE cluster
+	bash scripts/delete-gke-cluster.sh

tests/README.md

@@ -37,15 +37,19 @@ make
 ```
 
 ```text
+build-images-with-plus         Build NGF and NGINX Plus images
 build-images                   Build NGF and NGINX images
 cleanup-gcp                    Cleanup all GCP resources
 cleanup-router                 Delete the GKE router
 cleanup-vm                     Delete the test GCP VM and delete the firewall rule
 create-and-setup-vm            Create and setup a GCP VM for tests
+create-gke-cluster             Create a GKE cluster
 create-gke-router              Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
 create-kind-cluster            Create a kind cluster
+delete-gke-cluster             Delete the GKE cluster
 delete-kind-cluster            Delete kind cluster
 help                           Display this help
+load-images-with-plus          Load NGF and NGINX Plus images on configured kind cluster
 load-images                    Load NGF and NGINX images on configured kind cluster
 run-tests-on-vm                Run the tests on a GCP VM
 setup-gcp-and-run-tests        Create and setup a GKE router and GCP VM for tests and run the tests
@@ -71,7 +75,9 @@ test                           Run the system tests against your default k8s clu
 
 ## Step 1 - Create a Kubernetes cluster
 
-This can be done in a cloud provider of choice, or locally using `kind`:
+This can be done in a cloud provider of choice, or locally using `kind`.
+
+To create a local `kind` cluster:
 
 ```makefile
 make create-kind-cluster
@@ -84,6 +90,17 @@ make create-kind-cluster
 make create-kind-cluster KIND_IMAGE=kindest/node:v1.27.3
 ```
 
+To create a GKE cluster:
+
+Before running the below `make` command, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
+required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions,
+and `GKE_NODES_SERVICE_ACCOUNT` needs to be the name of a service account that has `Artifact Registry Reader`,
+`Kubernetes Engine Node Service Account` and `Monitoring Viewer` permissions.
+
+```makefile
+make create-gke-cluster
+```
+
 ## Step 2 - Build and Load Images
 
 Loading the images only applies to a `kind` cluster. If using a cloud provider, you will need to tag and push
@@ -115,7 +132,7 @@ make test TAG=$(whoami) PLUS_ENABLED=true
 
 ### 3b - Run the tests on a GKE cluster from a GCP VM
 
-This step only applies if you would like to run the tests from a GCP based VM.
+This step only applies if you would like to run the tests on a GKE cluster from a GCP based VM.
 
 Before running the below `make` command, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
 required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
@@ -205,7 +222,7 @@ For more information of filtering specs, see [the docs here](https://onsi.github
     make delete-kind-cluster
     ```
 
-2. Delete the GCP components (GKE router, VM, and firewall rule), if required
+2. Delete the GCP components (GKE cluster, GKE router, VM, and firewall rule), if required
 
     ```makefile
     make cleanup-gcp
@@ -220,3 +237,7 @@ For more information of filtering specs, see [the docs here](https://onsi.github
     ```makefile
     make cleanup-vm
     ```
+
+    ```makefile
+    make delete-gke-cluster
+    ```

tests/scripts/cleanup-vm.sh

@@ -2,13 +2,15 @@
 
 source scripts/vars.env
 
+skip_gke_master_control_node_access="${1:-false}"
+
 # Remove VM IP from GKE master control node access, if required
-if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then
-    EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
+if [ ${ADD_VM_IP_AUTH_NETWORKS} = "true" ] && [ ${skip_gke_master_control_node_access} != "true" ]; then
+    EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
                     --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
-    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} \
+    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} \
                             --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//')
-    gcloud container clusters update ${GKE_CLUSTER_NAME} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK}
+    gcloud container clusters update ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK}
 fi
 
 gcloud compute instances delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE}

tests/scripts/create-and-setup-gcp-vm.sh

@@ -24,17 +24,17 @@ gcloud compute instances create ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone
 if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then
     EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
                     --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
-    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} \
+    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} \
                             --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//')
-    gcloud container clusters update ${GKE_CLUSTER_NAME} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK},${EXTERNAL_IP}/32
+    gcloud container clusters update ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${EXTERNAL_IP}/32,${CURRENT_AUTH_NETWORK}
 fi
 
 # Poll for SSH connectivity
-MAX_RETRIES=30
-RETRY_INTERVAL=10
+MAX_RETRIES=10
+RETRY_INTERVAL=5
 for ((i=1; i<=MAX_RETRIES; i++)); do
     echo "Attempt $i to connect to the VM..."
-    gcloud compute ssh ${RESOURCE_NAME} --zone=${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --quiet --command="echo 'VM is ready'"
+    gcloud compute ssh username@${RESOURCE_NAME} --zone=${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --quiet --command="echo 'VM is ready'"
     if [ $? -eq 0 ]; then
         echo "SSH connection successful. VM is ready."
         break
@@ -43,6 +43,6 @@ for ((i=1; i<=MAX_RETRIES; i++)); do
     sleep ${RETRY_INTERVAL}
 done
 
-gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env ${RESOURCE_NAME}:~
+gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env username@${RESOURCE_NAME}:~
 
-gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/install-deps.sh
+gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} username@${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/install-deps.sh

tests/scripts/create-gke-cluster.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+source scripts/vars.env
+
+ip_random_digit=$((1 + $RANDOM % 250))
+
+gcloud container clusters create ${GKE_CLUSTER_NAME} \
+    --project ${GKE_PROJECT} \
+    --zone ${GKE_CLUSTER_ZONE} \
+    --enable-master-authorized-networks \
+    --enable-ip-alias \
+    --service-account ${GKE_NODES_SERVICE_ACCOUNT} \
+    --enable-private-nodes \
+    --master-ipv4-cidr 172.16.${ip_random_digit}.32/28 \
+    --metadata=block-project-ssh-keys=TRUE

tests/scripts/delete-gke-cluster.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source scripts/vars.env
+
+gcloud container clusters delete ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --project ${GKE_PROJECT} --quiet

tests/scripts/remote-scripts/run-tests.sh

@@ -6,4 +6,4 @@ source ~/vars.env
 
 echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.profile && . $HOME/.profile
 
-cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true
+cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION}

tests/scripts/run-tests-gcp-vm.sh

@@ -4,8 +4,8 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 source scripts/vars.env
 
-gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env ${RESOURCE_NAME}:~
+gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env username@${RESOURCE_NAME}:~
 
-gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/run-tests.sh
+gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} username@${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/run-tests.sh
 
-gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse ${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
+gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .