improves chainsaw single-tenancy test times by running in parallel (#2654)

ChrisJBurns · web-flow · commit d783c01660ff · 2025-11-19T11:38:14.000Z
* tries to speed up tests

Signed-off-by: Chris Burns &lt;29541485+ChrisJBurns@users.noreply.github.com&gt;

* uses more templating

Signed-off-by: Chris Burns &lt;29541485+ChrisJBurns@users.noreply.github.com&gt;

* adds fixes

Signed-off-by: Chris Burns &lt;29541485+ChrisJBurns@users.noreply.github.com&gt;

---------

Signed-off-by: Chris Burns &lt;29541485+ChrisJBurns@users.noreply.github.com&gt;
diff --git a/.github/workflows/operator-ci.yml b/.github/workflows/operator-ci.yml
@@ -203,5 +203,5 @@ jobs:
         chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/test-scenarios --config .chainsaw.yaml
         chainsaw test --test-dir test/e2e/chainsaw/operator/multi-tenancy/cleanup --config .chainsaw.yaml
         chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/setup --config .chainsaw.yaml
-        chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios --config .chainsaw.yaml
+        chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios --parallel 10 --config .chainsaw.yaml
         chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/cleanup --config .chainsaw.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/assert-proxy-svc-loadbalancer-ip.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/assert-proxy-svc-loadbalancer-ip.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: mcp-yardstick-proxy-lb
+  name: (join('-', ['mcp', $testPrefix, 'proxy-lb']))
   namespace: toolhive-system
 spec:
   type: LoadBalancer
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/mcp_serviceaccount.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/mcp_serviceaccount.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxy-svc-loadbalancer.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxy-svc-loadbalancer.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: mcp-yardstick-proxy-lb
+  name: (join('-', ['mcp', $testPrefix, 'proxy-lb']))
   namespace: toolhive-system
 spec:
   type: LoadBalancer
@@ -13,4 +13,4 @@ spec:
   selector:
     app: mcpserver
     app.kubernetes.io/name: mcpserver
-    app.kubernetes.io/instance: yardstick
+    app.kubernetes.io/instance: ($testPrefix)
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxyrunner-role.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxyrunner-role.yaml
@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: yardstick-proxy-runner
+  name: (join('-', [$testPrefix, 'proxy-runner']))
   namespace: toolhive-system
 rules:
 - apiGroups:
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxyrunner-rolebinding.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxyrunner-rolebinding.yaml
@@ -1,13 +1,13 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: yardstick-proxy-runner
+  name: (join('-', [$testPrefix, 'proxy-runner']))
   namespace: toolhive-system
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: yardstick-proxy-runner
+  name: (join('-', [$testPrefix, 'proxy-runner']))
 subjects:
 - kind: ServiceAccount
-  name: yardstick-proxy-runner
+  name: (join('-', [$testPrefix, 'proxy-runner']))
   namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxyrunner-serviceaccount.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/common/proxyrunner-serviceaccount.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: yardstick-proxy-runner
+  name: (join('-', [$testPrefix, 'proxy-runner']))
   namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-headless-svc.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-headless-svc.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: mcp-yardstick-headless
+  name: mcp-st-sse-headless
   namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-pod-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-pod-running.yaml
@@ -1,11 +1,11 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: yardstick
+  name: st-sse
   namespace: toolhive-system
 spec:
   template:
     spec:
-      serviceAccountName: test-mcp-sa
+      serviceAccountName: (join('-', [$testPrefix, 'sa']))
 status:
   availableReplicas: 1
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-proxy-runner-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-proxy-runner-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: yardstick
+  name: st-sse
   namespace: toolhive-system
 spec:
   replicas: 1
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-proxy-runner-svc.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-proxy-runner-svc.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: mcp-yardstick-proxy
+  name: mcp-st-sse-proxy
   namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/assert-mcpserver-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: toolhive.stacklok.dev/v1alpha1
 kind: MCPServer
 metadata:
-  name: yardstick
+  name: st-sse
   namespace: toolhive-system
 status:
   message: "MCP server is running"
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/chainsaw-test.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/chainsaw-test.yaml
@@ -1,28 +1,31 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: sse-mcp-server
+  name: st-sse
 spec:
   description: Deploys SSE MCP server and verifies it's running
   timeouts:
     apply: 30s
     assert: 60s
     cleanup: 30s
     exec: 300s
+  template: true
+  bindings:
+    - name: testPrefix
+      value: "st-sse"
   steps:
   - name: verify-operator
     description: Ensure operator is ready before testing
     try:
     - assert:
         file: ../../setup/assert-operator-ready.yaml
-      
   - name: deploy-mcpserver
     description: Deploy a basic MCPServer instance and verify it's ready. It also creates a service account and tells the MCP Server to use it.
     try:
     - apply:
-        file: ../common/mcp_serviceaccount.yaml
+        file: serviceaccount.yaml
     - assert:
-        file: ../common/mcp_serviceaccount.yaml
+        file: serviceaccount.yaml
     - apply:
         file: mcpserver.yaml
     - assert:
@@ -43,52 +46,6 @@ spec:
         file: ../common/proxyrunner-rolebinding.yaml
     - assert:
         file: ../common/proxyrunner-serviceaccount.yaml
-    # Validate that ConfigMap is created with runconfig.json content
-    - script:
-        content: |
-          echo "Verifying ConfigMap creation and content..."
-          
-          # Wait for ConfigMap to be created
-          for i in $(seq 1 10); do
-            if kubectl get configmap yardstick-runconfig -n toolhive-system >/dev/null 2>&1; then
-              echo "✓ ConfigMap yardstick-runconfig exists"
-              break
-            fi
-            echo "  Waiting for ConfigMap... (attempt $i/10)"
-            sleep 2
-          done
-          
-          # Verify ConfigMap contains runconfig.json with proper content
-          CONFIGMAP_JSON=$(kubectl get configmap yardstick-runconfig -n toolhive-system -o jsonpath='{.data.runconfig\.json}' 2>/dev/null || echo "")
-          
-          if [ -z "$CONFIGMAP_JSON" ]; then
-            echo "✗ ConfigMap does not contain runconfig.json data"
-            kubectl get configmap yardstick-runconfig -n toolhive-system -o yaml
-            exit 1
-          fi
-          
-          echo "✓ ConfigMap contains runconfig.json data"
-          
-          # Validate JSON structure contains expected fields
-          if echo "$CONFIGMAP_JSON" | jq -e '.schema_version and .image and .name and .transport' > /dev/null 2>&1; then
-            echo "✓ runconfig.json contains required fields (schema_version, image, name, transport)"
-          else
-            echo "✗ runconfig.json missing required fields"
-            echo "ConfigMap content:"
-            echo "$CONFIGMAP_JSON"
-            exit 1
-          fi
-          
-          # Verify transport matches expected value (sse)
-          TRANSPORT=$(echo "$CONFIGMAP_JSON" | jq -r '.transport' 2>/dev/null)
-          if [ "$TRANSPORT" = "sse" ]; then
-            echo "✓ runconfig.json transport is correctly set to 'sse'"
-          else
-            echo "✗ runconfig.json transport is '$TRANSPORT', expected 'sse'"
-            exit 1
-          fi
-          
-          echo "✅ ConfigMap validation passed!"
     - apply:
         file: ../common/proxy-svc-loadbalancer.yaml
     - assert:
@@ -98,15 +55,18 @@ spec:
     description: Test the SSE->SSE MCP server by sending requests at the toolhive proxy
     try:
     - script:
+        env:
+          - name: loadBalancerServiceName
+            value: (join('-', ['mcp', $testPrefix, 'proxy-lb']))
         content: |
           # Get LoadBalancer external IP
-          echo "Getting LoadBalancer external IP for service mcp-yardstick-proxy-lb..."
+          echo "Getting LoadBalancer external IP for service $loadBalancerServiceName..."
           
-          EXTERNAL_IP=$(kubectl get svc mcp-yardstick-proxy-lb -n toolhive-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
+          EXTERNAL_IP=$(kubectl get svc $loadBalancerServiceName -n toolhive-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
           
           if [ -z "$EXTERNAL_IP" ]; then
             echo "LoadBalancer did not get external IP within timeout"
-            kubectl describe svc mcp-yardstick-proxy-lb -n toolhive-system
+            kubectl describe svc $loadBalancerServiceName -n toolhive-system
             exit 1
           fi
           
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/mcpserver.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/mcpserver.yaml
@@ -1,12 +1,12 @@
 apiVersion: toolhive.stacklok.dev/v1alpha1
 kind: MCPServer
 metadata:
-  name: yardstick
+  name: ($testPrefix)
   namespace: toolhive-system
 spec:
   image: ghcr.io/stackloklabs/yardstick/yardstick-server:0.0.2
   transport: sse
-  serviceAccount: test-mcp-sa
+  serviceAccount: (join('-', [$testPrefix, 'sa']))
   env:
   - name: TRANSPORT
     value: sse
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/serviceaccount.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/sse/serviceaccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: (join('-', [$testPrefix, 'sa']))
+  namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-pod-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-pod-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: yardstick
+  name: st-stdio-streamable-http
   namespace: toolhive-system
 status:
   availableReplicas: 1
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-proxy-runner-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-proxy-runner-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: yardstick
+  name: st-stdio-streamable-http
   namespace: toolhive-system
 spec:
   replicas: 1
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-proxy-runner-svc.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-proxy-runner-svc.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: mcp-yardstick-proxy
+  name: mcp-st-stdio-streamable-http-proxy
   namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/assert-mcpserver-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: toolhive.stacklok.dev/v1alpha1
 kind: MCPServer
 metadata:
-  name: yardstick
+  name: st-stdio-streamable-http
   namespace: toolhive-system
 status:
   message: "MCP server is running"
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/chainsaw-test.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/chainsaw-test.yaml
@@ -1,21 +1,24 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: stdio-streamable-http-mcp-server
+  name: st-stdio-streamable-http
 spec:
   description: Deploys STDIO MCP server with streamable-http proxy mode and verifies it's running
   timeouts:
     apply: 30s
     assert: 60s
     cleanup: 30s
     exec: 300s
+  template: true
+  bindings:
+    - name: testPrefix
+      value: "st-stdio-streamable-http"
   steps:
   - name: verify-operator
     description: Ensure operator is ready before testing
     try:
     - assert:
         file: ../../setup/assert-operator-ready.yaml
-      
   - name: deploy-mcpserver
     description: Deploy a basic MCPServer instance and verify it's ready
     try:
@@ -44,15 +47,18 @@ spec:
     description: Test the streamable-http->STDIO MCP server by sending requests at the toolhive proxy
     try:
     - script:
+        env:
+          - name: loadBalancerServiceName
+            value: (join('-', ['mcp', $testPrefix, 'proxy-lb']))
         content: |
           # Get LoadBalancer external IP
-          echo "Getting LoadBalancer external IP for service mcp-yardstick-proxy-lb..."
+          echo "Getting LoadBalancer external IP for service $loadBalancerServiceName..."
 
-          EXTERNAL_IP=$(kubectl get svc mcp-yardstick-proxy-lb -n toolhive-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
+          EXTERNAL_IP=$(kubectl get svc $loadBalancerServiceName -n toolhive-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
           
           if [ -z "$EXTERNAL_IP" ]; then
             echo "LoadBalancer did not get external IP within timeout"
-            kubectl describe svc mcp-yardstick-proxy-lb -n toolhive-system
+            kubectl describe svc $loadBalancerServiceName -n toolhive-system
             exit 1
           fi
           
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/mcpserver.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio-streamable-http/mcpserver.yaml
@@ -1,7 +1,7 @@
 apiVersion: toolhive.stacklok.dev/v1alpha1
 kind: MCPServer
 metadata:
-  name: yardstick
+  name: ($testPrefix)
   namespace: toolhive-system
 spec:
   image: ghcr.io/stackloklabs/yardstick/yardstick-server:0.0.2
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-pod-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-pod-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: yardstick
+  name: st-stdio
   namespace: toolhive-system
 status:
   availableReplicas: 1
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-proxy-runner-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-proxy-runner-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: yardstick
+  name: st-stdio
   namespace: toolhive-system
 spec:
   replicas: 1
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-proxy-runner-svc.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-proxy-runner-svc.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: mcp-yardstick-proxy
+  name: mcp-st-stdio-proxy
   namespace: toolhive-system
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/assert-mcpserver-running.yaml
@@ -1,7 +1,7 @@
 apiVersion: toolhive.stacklok.dev/v1alpha1
 kind: MCPServer
 metadata:
-  name: yardstick
+  name: st-stdio
   namespace: toolhive-system
 status:
   message: "MCP server is running"
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/chainsaw-test.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/chainsaw-test.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/mcpserver.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/stdio/mcpserver.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-headless-svc.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-headless-svc.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-pod-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-pod-running.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-proxy-runner-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-proxy-runner-running.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-proxy-runner-svc.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-proxy-runner-svc.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-running.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/assert-mcpserver-running.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/chainsaw-test.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/chainsaw-test.yaml
diff --git a/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/mcpserver.yaml b/test/e2e/chainsaw/operator/single-tenancy/test-scenarios/streamable-http/mcpserver.yaml
