Add e2e tests for Prefix

Author: henrybear327

.github/env

@@ -0,0 +1,3 @@
+golang-version=1.23
+kind-version=v0.25.0
+kind-image=kindest/node:v1.31.2

.github/workflows/e2e-tests.yaml

@@ -0,0 +1,77 @@
+# Modified from https://github.com/prometheus-operator/prometheus-operator/blob/main/.github/workflows/e2e-feature-gated.yaml
+name: e2e-tests
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+  pull_request:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+env:
+  NETBOX_HOST: demo.netbox.dev
+  AUTH_TOKEN: 0123456789abcdef0123456789abcdef01234567
+  POD_NAMESPACE: default
+  HTTPS_ENABLE: true
+  NETBOX_RESTORATION_HASH_FIELD_NAME: netboxOperatorRestorationHash
+jobs:
+  e2e-tests:
+    name: E2E tests for netbox operator
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      with: 
+        go-version: 1.23.4
+    - name: Install Chainsaw
+      uses: kyverno/action-install-chainsaw@f2b47b97dc889c12702113753d713f01ec268de5 #v0.2.12
+    - name: Check Chainsaw installation
+      run: chainsaw version
+    - name: Import environment variables from file
+      run: |
+        cat ".github/env" >> "$GITHUB_ENV"
+        echo "E2E_DIAGNOSTIC_DIRECTORY=$(mktemp -d)" >> "$GITHUB_ENV"
+    - name: Start kind cluster
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+      with:
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./tests/e2e/kind-config.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        echo "Waiting for all nodes to be ready..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=120s
+        kubectl get nodes
+        echo "Waiting for all pods to be ready..."
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl get pods -A
+        echo "Cluster information"
+        kubectl cluster-info
+    - name: Setup kind cluster with required software such as NetBox
+      run: |
+        make create-kind
+    - name: Deploy NetBox operator to the kind cluster
+      run: |
+        make deploy-kind
+    - name: Run tests
+      env:
+        E2E_DIAGNOSTIC_DIRECTORY: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }}
+      run: |
+        # # Very straight forward way of implementing a test and checking the result
+        # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml
+        # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim
+        # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample
+
+        # Use Chainsaw
+        make test-e2e
+    - name: Upload diagnostics artifact
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      with:
+        name: cluster-state
+        path: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }}
+        retention-days: 15

.gitignore

@@ -9,6 +9,7 @@ Dockerfile.cross
 
 # Test binary, built with `go test -c`
 *.test
+database.sql
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out

CONTRIBUTING.md

@@ -26,14 +26,13 @@ To optionally access the NetBox UI:
 - Port-forward NetBox: `kubectl port-forward deploy/netbox 8080:8080 -n default`
 - Open <http://localhost:8080> in your favorite browser and log in with the username `admin` and password `admin`, you will be able to access the local NetBox instance running in the kind cluster.
 - Open a new terminal window and export the following environment variables:
-
-    ```bash
-    export NETBOX_HOST="localhost:8080"
-    export AUTH_TOKEN="0123456789abcdef0123456789abcdef01234567"
-    export POD_NAMESPACE="default"
-    export HTTPS_ENABLE="false"
-    export NETBOX_RESTORATION_HASH_FIELD_NAME="netboxOperatorRestorationHash"
-    ```
+  ```bash
+  export NETBOX_HOST="localhost:8080"
+  export AUTH_TOKEN="0123456789abcdef0123456789abcdef01234567"
+  export POD_NAMESPACE="default"
+  export HTTPS_ENABLE="false"
+  export NETBOX_RESTORATION_HASH_FIELD_NAME="netboxOperatorRestorationHash"
+  ```
 
 - Run the NetBox Operator locally `make install && make run`
 
@@ -46,14 +45,13 @@ Note: This requires a running NetBox instance that you can use (e.g. <https://de
   - Open <https://demo.netbox.dev/user/api-tokens/> and create a token "0123456789abcdef0123456789abcdef01234567" with default settings
   - Open <https://demo.netbox.dev/extras/custom-fields/add/> and create a custom field called "netboxOperatorRestorationHash" for Object types "IPAM > IP Address" and "IPAM > Prefix"
 - Open a new terminal window and export the following environment variables:
-
-    ```bash
-    export NETBOX_HOST="demo.netbox.dev"
-    export AUTH_TOKEN="0123456789abcdef0123456789abcdef01234567"
-    export POD_NAMESPACE="default"
-    export HTTPS_ENABLE="true"
-    export NETBOX_RESTORATION_HASH_FIELD_NAME="netboxOperatorRestorationHash"
-    ```
+  ```bash
+  export NETBOX_HOST="demo.netbox.dev"
+  export AUTH_TOKEN="0123456789abcdef0123456789abcdef01234567"
+  export POD_NAMESPACE="default"
+  export HTTPS_ENABLE="true"
+  export NETBOX_RESTORATION_HASH_FIELD_NAME="netboxOperatorRestorationHash"
+  ```
 
 - Run the NetBox Operator locally `make install && make run`
 
@@ -130,3 +128,9 @@ make undeploy
 Run `make help` for more information on all potential `make` targets
 
 More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
+
+## Running e2e tests locally
+
+Please read the [README in the e2e test directory] for more information!
+
+[README in the e2e test directory]: ./tests/e2e/README.md

Makefile

@@ -226,3 +226,7 @@ $(ENVTEST): $(LOCALBIN)
 generate_mocks: # TODO: auto install go install go.uber.org/mock/mockgen@latest
 	mkdir -p ${GEN_DIR}
 	mockgen -destination ${GEN_DIR}/${NETBOX_MOCKS_OUTPUT_FILE} -source=${INTERFACE_DEFITIONS_DIR}
+
+.PHONY: test-e2e
+test-e2e: # notice that if we apply several CRs at the same time, the IP/Prefix/etc. allocated to the CR might differ each time due to the timing of execution for reservation on the NetBox side
+	chainsaw test --namespace e2e

README.md

@@ -64,7 +64,6 @@ Please read [ParentPrefixSelector guide] for more information!
 
 [ParentPrefixSelector guide]: ./ParentPrefixSelectorGuide.md
 
-
 # Project Distribution
 
 Following are the steps to build the installer and distribute this project to users.

kind/load-data-job/local-demo-data.sql

@@ -63,6 +63,14 @@ VALUES ('2024-06-14 09:57:11.709344+00', '2024-06-14 09:57:11.709359+00', '{"cus
 INSERT INTO public.ipam_prefix (created, last_updated, custom_field_data, prefix, status, is_pool, description, role_id, site_id, tenant_id, vlan_id, vrf_id, _children, _depth, mark_utilized, comments)
 VALUES ('2024-06-14 10:01:10.094083+00', '2024-06-14 10:01:10.094095+00', '{}', '2.0.0.0/16', 'active', false, '', NULL, NULL, 100, NULL, NULL, 0, 0, false, '');
 
+-- 2.1.0.0/24
+INSERT INTO public.ipam_prefix (created, last_updated, custom_field_data, prefix, status, is_pool, description, role_id, site_id, tenant_id, vlan_id, vrf_id, _children, _depth, mark_utilized, comments)
+VALUES ('2024-06-14 10:01:10.094083+00', '2024-06-14 10:01:10.094095+00', '{}', '2.1.0.0/24', 'active', false, '', NULL, NULL, 100, NULL, NULL, 0, 0, false, '');
+
+-- 2.2.0.0/24
+INSERT INTO public.ipam_prefix (created, last_updated, custom_field_data, prefix, status, is_pool, description, role_id, site_id, tenant_id, vlan_id, vrf_id, _children, _depth, mark_utilized, comments)
+VALUES ('2024-06-14 10:01:10.094083+00', '2024-06-14 10:01:10.094095+00', '{}', '2.2.0.0/24', 'active', false, '', NULL, NULL, 100, NULL, NULL, 0, 0, false, '');
+
 -- 3.0.0.0/24 - 3.0.8.0/24 (watch out for the upper/lower-case)
 -- Pool 1, Production (IPv4)
 INSERT INTO public.ipam_prefix (created, last_updated, custom_field_data, prefix, status, is_pool, description, role_id, site_id, tenant_id, vlan_id, vrf_id, _children, _depth, mark_utilized, comments)

kind/local-env.sh

@@ -51,4 +51,3 @@ helm upgrade --install --namespace="${NAMESPACE}" netbox \
   https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.161/netbox-5.0.0-beta.161.tgz
 
 kubectl rollout status --namespace="${NAMESPACE}" deployment netbox
-

tests/e2e/Prefix/IPv4/GivenPrefixClaimWhenAppliedThenFailedPrefixExhausted/chainsaw-test.yaml

@@ -0,0 +1,95 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: prefixclaim-ipv4-apply-exhausted
+spec:
+  steps:
+    - name: Install CR 1
+      description: Apply prefix claim CR 1
+      try:
+        - apply:
+            file: netbox_v1_prefixclaim_1.yaml
+        - assert:
+            resource:
+              apiVersion: netbox.dev/v1
+              kind: PrefixClaim
+              metadata:
+                name: prefixclaim-ipv4-apply-prefixexhausted-1
+              spec:
+                comments: your comments
+                description: some description
+                parentPrefix: 2.1.0.0/24
+                prefixLength: /25
+                preserveInNetbox: false
+                site: DM-Akron
+                tenant: Dunder-Mifflin, Inc.
+              status:
+                parentPrefix: 2.1.0.0/24
+                prefix: 2.1.0.0/25
+                prefixName: prefixclaim-ipv4-apply-prefixexhausted-1
+        - assert:
+            resource:
+              apiVersion: netbox.dev/v1
+              kind: Prefix
+              metadata:
+                name: prefixclaim-ipv4-apply-prefixexhausted-1
+              spec:
+                comments: your comments
+                description: some description
+                prefix: 2.1.0.0/25
+                site: DM-Akron
+                tenant: Dunder-Mifflin, Inc.
+    - name: Install CR 2
+      description: Apply prefix claim CR 2
+      try:
+        - apply:
+            file: netbox_v1_prefixclaim_2.yaml
+        - assert:
+            resource:
+              apiVersion: netbox.dev/v1
+              kind: PrefixClaim
+              metadata:
+                name: prefixclaim-ipv4-apply-prefixexhausted-2
+              spec:
+                comments: your comments
+                description: some description
+                parentPrefix: 2.1.0.0/24
+                prefixLength: /25
+                preserveInNetbox: false
+                site: DM-Akron
+                tenant: Dunder-Mifflin, Inc.
+              status:
+                parentPrefix: 2.1.0.0/24
+                prefix: 2.1.0.128/25
+                prefixName: prefixclaim-ipv4-apply-prefixexhausted-2
+        - assert:
+            resource:
+              apiVersion: netbox.dev/v1
+              kind: Prefix
+              metadata:
+                name: prefixclaim-ipv4-apply-prefixexhausted-2
+              spec:
+                comments: your comments
+                description: some description
+                prefix: 2.1.0.128/25
+                site: DM-Akron
+                tenant: Dunder-Mifflin, Inc.
+    - name: Install CR 3
+      description: Apply prefix claim CR 3
+      try:
+        - apply:
+            file: netbox_v1_prefixclaim_3.yaml
+        - assert:
+            resource:
+              apiVersion: v1
+              count: 1
+              kind: Event
+              type: Warning
+              reason: PrefixCRNotCreated
+              source:
+                component: prefix-claim-controller
+              message: Failed to fetch new Prefix from NetBox. parent prefix exhausted, will restart the parent prefix selection process
+              involvedObject:
+                apiVersion: netbox.dev/v1
+                kind: PrefixClaim
+                name: prefixclaim-ipv4-apply-prefixexhausted-3

tests/e2e/Prefix/IPv4/GivenPrefixClaimWhenAppliedThenFailedPrefixExhausted/netbox_v1_prefixclaim_1.yaml

@@ -0,0 +1,15 @@
+apiVersion: netbox.dev/v1
+kind: PrefixClaim
+metadata:
+  labels:
+    app.kubernetes.io/name: netbox-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: prefixclaim-ipv4-apply-prefixexhausted-1
+spec:
+  tenant: "Dunder-Mifflin, Inc."
+  site: "DM-Akron"
+  description: "some description"
+  comments: "your comments"
+  preserveInNetbox: false
+  parentPrefix: "2.1.0.0/24"
+  prefixLength: "/25"