Add istio bug report on failure (#9458)

* add istioctl bug report * add debug, clean up istio commands * bump curl, add check httpbin is ready * use hashFiles exists * add resource req limits * pr feedback * trigger failure to get resource limits in oss * set istio version * set istio version in github workflow * use GeneratedFiles.TempDir * use FailureDir * remove fail() to test artifact upload * switch istioctl back to testInstallation * remove increased timeout --------- Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
solo-io · May 9, 2024 · f55d646 · f55d646
1 parent a4e8834
commit f55d646
Show file tree

Hide file tree

Showing 18 changed files with 295 additions and 169 deletions.
diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml
@@ -6,6 +6,10 @@ on:
 env:
   VERSION: '1.0.0-ci1'
   GITHUB_TOKEN: ${{ github.token }}
+# The istio version to use for the e2e tests
+# TODO(npolshak): Test across multiple versions of Istio
+# https://istio.io/latest/docs/releases/supported-releases/#support-status-of-istio-releases
+  ISTIO_VERSION: "1.19.9"
 
 jobs:
   prepare_env:
@@ -95,4 +99,9 @@ jobs:
         cluster-name: kind
         test-args: ${{ matrix.test.go-test-args }}
         run-regex: ${{ matrix.test.go-test-run-regex }}
-
+    - name: Archive bug report directory on failure
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: bug-report
+        path: ./_test/bug_report
diff --git a/changelog/v1.17.0-beta26/add-istio-bug-report-on-failure.yaml b/changelog/v1.17.0-beta26/add-istio-bug-report-on-failure.yaml
@@ -0,0 +1,6 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6048
+    resolvesIssue: false
+    description: >-
+      Add istioctl bug-report to the end-to-end tests on failure.
diff --git a/test/kubernetes/e2e/README.md b/test/kubernetes/e2e/README.md
@@ -23,6 +23,12 @@ This package holds the entry point for each of our `TestInstallation`.
 
 See [Load balancing tests](./load_balancing_tests.md) for more information about how these tests are run in CI.
 
+## Environment Variables
+
+Some tests may require environment variables to be set. Some required env vars are:
+
+- Istio features: Require `ISTIO_VERSION` to be set. The tests running in CI use `ISTIO_VERSION="${ISTIO_VERSION:-1.19.9}"` to default to a specific version of Istio.
+
 ## Debugging
 
 Refer to the [Debugging guide](./debugging.md) for more information on how to debug tests.

diff --git a/test/kubernetes/e2e/features/headless_svc/testdata/setup.yaml b/test/kubernetes/e2e/features/headless_svc/testdata/setup.yaml
@@ -46,4 +46,9 @@ spec:
       command:
         - "tail"
         - "-f"
-        - "/dev/null"
+        - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/features/istio/auto_mtls_suite.go b/test/kubernetes/e2e/features/istio/auto_mtls_suite.go
@@ -3,6 +3,7 @@ package istio
 import (
 	"context"
 
+	"github.com/onsi/gomega"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/solo-io/gloo/pkg/utils/kubeutils"
@@ -31,6 +32,8 @@ func NewIstioAutoMtlsSuite(ctx context.Context, testInst *e2e.TestInstallation)
 func (s *istioAutoMtlsTestingSuite) SetupSuite() {
 	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest)
 	s.NoError(err, "can apply setup manifest")
+	// Check that istio injection is successful and httpbin is running
+	s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, httpbinDeployment.ObjectMeta, gomega.Equal(1))
 
 	// Ensure that the proxy service and deployment are created
 	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, k8sRoutingManifest)
@@ -45,6 +48,7 @@ func (s *istioAutoMtlsTestingSuite) TearDownSuite() {
 
 	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest)
 	s.NoError(err, "can delete setup manifest")
+	s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, httpbinDeployment)
 }
 
 func (s *istioAutoMtlsTestingSuite) TestMtlsStrictPeerAuth() {
@@ -64,7 +68,8 @@ func (s *istioAutoMtlsTestingSuite) TestMtlsStrictPeerAuth() {
 			curl.WithHostHeader("httpbin"),
 			curl.WithPath("/headers"),
 		},
-		expectedMtlsResponse)
+		expectedMtlsResponse,
+	)
 }
 
 func (s *istioAutoMtlsTestingSuite) TestMtlsPermissivePeerAuth() {

diff --git a/test/kubernetes/e2e/features/istio/inputs/setup.yaml b/test/kubernetes/e2e/features/istio/inputs/setup.yaml
@@ -59,18 +59,11 @@ spec:
             - "600s" # override default 10s
           ports:
             - containerPort: 8080
-        - name: curl
-          image: curlimages/curl:7.83.1
           resources:
             requests:
               cpu: "100m"
             limits:
               cpu: "200m"
-          imagePullPolicy: IfNotPresent
-          command:
-            - "tail"
-            - "-f"
-            - "/dev/null"
 ---
 apiVersion: v1
 kind: Namespace
@@ -93,4 +86,9 @@ spec:
       command:
         - "tail"
         - "-f"
-        - "/dev/null"
+        - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/features/istio/no_mtls_suite.go b/test/kubernetes/e2e/features/istio/no_mtls_suite.go
@@ -3,6 +3,7 @@ package istio
 import (
 	"context"
 
+	"github.com/onsi/gomega"
 	"github.com/solo-io/gloo/pkg/utils/kubeutils"
 	"github.com/solo-io/gloo/pkg/utils/requestutils/curl"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
@@ -31,6 +32,8 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.
 func (s *istioTestingSuite) SetupSuite() {
 	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest)
 	s.NoError(err, "can apply setup manifest")
+	// Check that istio injection is successful and httpbin is running
+	s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, httpbinDeployment.ObjectMeta, gomega.Equal(1))
 
 	// Ensure that the proxy service and deployment are created
 	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, k8sRoutingManifest)
@@ -45,6 +48,8 @@ func (s *istioTestingSuite) TearDownSuite() {
 
 	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest)
 	s.NoError(err, "can delete setup manifest")
+	s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, httpbinDeployment)
+
 }
 
 func (s *istioTestingSuite) TestStrictPeerAuth() {
@@ -65,7 +70,8 @@ func (s *istioTestingSuite) TestStrictPeerAuth() {
 			curl.WithHostHeader("httpbin"),
 			curl.WithPath("/headers"),
 		},
-		expectedServiceUnavailableResponse)
+		expectedServiceUnavailableResponse,
+	)
 }
 
 func (s *istioTestingSuite) TestPermissivePeerAuth() {

diff --git a/test/kubernetes/e2e/features/istio/types.go b/test/kubernetes/e2e/features/istio/types.go
@@ -28,6 +28,14 @@ var (
 	proxyDeployment = &appsv1.Deployment{ObjectMeta: glooProxyObjectMeta}
 	proxyService    = &corev1.Service{ObjectMeta: glooProxyObjectMeta}
 
+	// httpbinDeployment is the Deployment that is in the Istio mesh
+	httpbinDeployment = &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "httpbin",
+			Namespace: "httpbin",
+		},
+	}
+
 	// curlPod is the Pod that will be used to execute curl requests, and is defined in the fault injection manifest files
 	curlPodExecOpt = kubectl.PodExecOptions{
 		Name:      "curl",

diff --git a/test/kubernetes/e2e/features/port_routing/testdata/setup.yaml b/test/kubernetes/e2e/features/port_routing/testdata/setup.yaml
@@ -47,4 +47,9 @@ spec:
       command:
         - "tail"
         - "-f"
-        - "/dev/null"
+        - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/features/route_options/testdata/fault-injection-filter-extension.yaml b/test/kubernetes/e2e/features/route_options/testdata/fault-injection-filter-extension.yaml
@@ -90,4 +90,9 @@ spec:
       command:
         - "tail"
         - "-f"
-        - "/dev/null"
+        - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/features/route_options/testdata/fault-injection-targetref.yaml b/test/kubernetes/e2e/features/route_options/testdata/fault-injection-targetref.yaml
@@ -88,4 +88,9 @@ spec:
       command:
         - "tail"
         - "-f"
-        - "/dev/null"
+        - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/features/upstreams/inputs/route-with-upstream.yaml b/test/kubernetes/e2e/features/upstreams/inputs/route-with-upstream.yaml
@@ -63,4 +63,9 @@ spec:
       command:
         - "tail"
         - "-f"
-        - "/dev/null"
+        - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/features/virtualhost_options/testdata/setup.yaml b/test/kubernetes/e2e/features/virtualhost_options/testdata/setup.yaml
@@ -80,3 +80,8 @@ spec:
         - "tail"
         - "-f"
         - "/dev/null"
+      resources:
+        requests:
+          cpu: "100m"
+        limits:
+          cpu: "200m"
diff --git a/test/kubernetes/e2e/k8sgateway/automtls_istio_test.go b/test/kubernetes/e2e/k8sgateway/automtls_istio_test.go
@@ -40,6 +40,9 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) {
 	t.Cleanup(func() {
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
+
+			// Generate istioctl bug report
+			testInstallation.CreateIstioBugReport(ctx)
 		}
 
 		testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error {

diff --git a/test/kubernetes/e2e/k8sgateway/istio_test.go b/test/kubernetes/e2e/k8sgateway/istio_test.go
@@ -40,6 +40,9 @@ func TestK8sGatewayIstio(t *testing.T) {
 	t.Cleanup(func() {
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
+
+			// Generate istioctl bug report
+			testInstallation.CreateIstioBugReport(ctx)
 		}
 
 		testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error {