ci: fix unstable t/chaos/killchaos

.github/workflows/chaos.yml

@@ -22,7 +22,16 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v2.1.3
         with:
-          go-version: "1.14"
+          go-version: "1.16"
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
 
       - name: Creating minikube cluster
         run: |

t/chaos/killetcd/killetcd.go

@@ -96,8 +96,23 @@ var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
 	ginkgo.It("check if everything works", func() {
 		utils.SetRoute(e, httpexpect.Status2xx)
 		utils.GetRouteList(e, http.StatusOK)
-		time.Sleep(1 * time.Second)
-		utils.GetRoute(e, http.StatusOK)
+		var resp *httpexpect.Response
+
+		resp = utils.GetRouteIgnoreError(e)
+		// wait 1s seems not enough, wait some more time to make sure nothing goes wrong
+		if resp.Raw().StatusCode != http.StatusOK {
+			for i := range [60]int{} {
+				timeWait := fmt.Sprintf("wait for %ds\n", i)
+				fmt.Fprint(ginkgo.GinkgoWriter, timeWait)
+				resp = utils.GetRouteIgnoreError(e)
+				if resp.Raw().StatusCode != http.StatusOK {
+					time.Sleep(time.Second)
+				} else {
+					break
+				}
+			}
+		}
+		gomega.Ω(resp.Raw().StatusCode).Should(gomega.BeNumerically("==", http.StatusOK))
 		utils.TestPrometheusEtcdMetric(e, 1)
 	})
 

t/chaos/utils/utils.go

@@ -39,6 +39,7 @@ type httpTestCase struct {
 	Path              string
 	Body              string
 	Headers           map[string]string
+	IgnoreError       bool
 	ExpectStatus      int
 	ExpectBody        string
 	ExpectStatusRange httpexpect.StatusRange
@@ -69,6 +70,10 @@ func caseCheck(tc httpTestCase) *httpexpect.Response {
 	}
 
 	resp := req.Expect()
+	if tc.IgnoreError {
+		return resp
+	}
+
 	if tc.ExpectStatus != 0 {
 		resp.Status(tc.ExpectStatus)
 	}
@@ -84,39 +89,48 @@ func caseCheck(tc httpTestCase) *httpexpect.Response {
 	return resp
 }
 
-func SetRoute(e *httpexpect.Expect, expectStatusRange httpexpect.StatusRange) {
-	caseCheck(httpTestCase{
+func SetRoute(e *httpexpect.Expect, expectStatusRange httpexpect.StatusRange) *httpexpect.Response {
+	return caseCheck(httpTestCase{
 		E:       e,
 		Method:  http.MethodPut,
 		Path:    "/apisix/admin/routes/1",
 		Headers: map[string]string{"X-API-KEY": token},
 		Body: `{
-			 "uri": "/get",
-			 "plugins": {
-				 "prometheus": {}
-			 },
-			 "upstream": {
-				 "nodes": {
-					 "httpbin.default.svc.cluster.local:8000": 1
-				 },
-				 "type": "roundrobin"
-			 }
-		 }`,
+			"uri": "/get",
+			"plugins": {
+				"prometheus": {}
+			},
+			"upstream": {
+				"nodes": {
+					"httpbin.default.svc.cluster.local:8000": 1
+				},
+				"type": "roundrobin"
+			}
+		}`,
 		ExpectStatusRange: expectStatusRange,
 	})
 }
 
-func GetRoute(e *httpexpect.Expect, expectStatus int) {
-	caseCheck(httpTestCase{
+func GetRoute(e *httpexpect.Expect, expectStatus int) *httpexpect.Response {
+	return caseCheck(httpTestCase{
 		E:            e,
 		Method:       http.MethodGet,
 		Path:         "/get",
 		ExpectStatus: expectStatus,
 	})
 }
 
-func GetRouteList(e *httpexpect.Expect, expectStatus int) {
-	caseCheck(httpTestCase{
+func GetRouteIgnoreError(e *httpexpect.Expect) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:           e,
+		Method:      http.MethodGet,
+		Path:        "/get",
+		IgnoreError: true,
+	})
+}
+
+func GetRouteList(e *httpexpect.Expect, expectStatus int) *httpexpect.Response {
+	return caseCheck(httpTestCase{
 		E:            e,
 		Method:       http.MethodGet,
 		Path:         "/apisix/admin/routes",
@@ -126,17 +140,17 @@ func GetRouteList(e *httpexpect.Expect, expectStatus int) {
 	})
 }
 
-func DeleteRoute(e *httpexpect.Expect) {
-	caseCheck(httpTestCase{
+func DeleteRoute(e *httpexpect.Expect) *httpexpect.Response {
+	return caseCheck(httpTestCase{
 		E:       e,
 		Method:  http.MethodDelete,
 		Path:    "/apisix/admin/routes/1",
 		Headers: map[string]string{"X-API-KEY": token},
 	})
 }
 
-func TestPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) {
-	caseCheck(httpTestCase{
+func TestPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) *httpexpect.Response {
+	return caseCheck(httpTestCase{
 		E:          e,
 		Method:     http.MethodGet,
 		Path:       "/apisix/prometheus/metrics",