fix(ci): udp forward failed and missing pigz (#1929)

.github/workflows/e2e-test-ci.yml

@@ -95,6 +95,8 @@ jobs:
           chmod +x ./kind
           sudo mv kind /usr/local/bin
 
+          sudo apt install -y pigz
+
       - name: Build images
         env:
           TAG: dev
@@ -183,6 +185,7 @@ jobs:
       - name: Load images from cache
         run: |
           echo "loading docker images..."
+          sudo apt install -y pigz
           pigz -dc docker.tar.gz | docker load
           make push-images
 

Dockerfile

@@ -1,5 +1,3 @@
-#syntax=docker/dockerfile:1.2
-
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.

test/e2e/scaffold/k8s.go

@@ -756,7 +756,7 @@ func (s *Scaffold) EnsureNumEndpointsReady(t testing.TestingT, endpointsName str
 		t,
 		statusMsg,
 		20,
-		5*time.Second,
+		2*time.Second,
 		func() (string, error) {
 			endpoints, err := e.CoreV1().Endpoints(s.Namespace()).Get(context.Background(), endpointsName, metav1.GetOptions{})
 			if err != nil {
@@ -769,7 +769,7 @@ func (s *Scaffold) EnsureNumEndpointsReady(t testing.TestingT, endpointsName str
 			if readyNum == desired {
 				return "Service is now available", nil
 			}
-			return fmt.Sprintf("Endpoints not ready yet, expect %v, actual %v", desired, readyNum), nil
+			return "failed", fmt.Errorf("endpoints not ready yet, expect %v, actual %v", desired, readyNum)
 		},
 	)
 	ginkgo.GinkgoT().Log(message)
@@ -781,3 +781,22 @@ func (s *Scaffold) GetKubernetesClient() *kubernetes.Clientset {
 	assert.Nil(ginkgo.GinkgoT(), err, "get kubernetes client")
 	return client
 }
+
+func (s *Scaffold) RunKubectlAndGetOutput(args ...string) (string, error) {
+	return k8s.RunKubectlAndGetOutputE(ginkgo.GinkgoT(), s.kubectlOptions, args...)
+}
+
+func (s *Scaffold) RunDigDNSClientFromK8s(args ...string) (string, error) {
+	kubectlArgs := []string{
+		"run",
+		"dig",
+		"-i",
+		"--rm",
+		"--restart=Never",
+		"--image-pull-policy=IfNotPresent",
+		"--image=toolbelt/dig",
+		"--",
+	}
+	kubectlArgs = append(kubectlArgs, args...)
+	return s.RunKubectlAndGetOutput(kubectlArgs...)
+}

test/e2e/scaffold/test_backend.go

@@ -23,6 +23,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
+const (
+	CoreDNSDeployment = "coredns"
+)
+
 var (
 	_testBackendDeploymentTemplate = `
 apiVersion: apps/v1
@@ -120,11 +124,11 @@ spec:
       targetPort: 50053
   type: ClusterIP
 `
-	_udpDeployment = `
+	_udpDeployment = fmt.Sprintf(`
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: coredns
+  name: %s
 spec:
   replicas: 1
   selector:
@@ -146,13 +150,13 @@ spec:
         readinessProbe:
           tcpSocket:
             port: 53
-          initialDelaySeconds: 5
+          initialDelaySeconds: 2
           periodSeconds: 10
         ports:    
         - name: dns
           containerPort: 53
           protocol: UDP
-`
+`, CoreDNSDeployment)
 	_udpService = `
 kind: Service
 apiVersion: v1

test/e2e/suite-gateway/gateway_udproute.go

@@ -17,8 +17,8 @@
 package gateway
 
 import (
-	"context"
 	"fmt"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
@@ -48,9 +48,12 @@ spec:
 		assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixStreamRoutesCreated(1), "Checking number of streamroute")
 		assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "Checking number of upstream")
 		// test dns query
-		r := s.DNSResolver()
-		host := "httpbin.org"
-		_, err = r.LookupIPAddr(context.Background(), host)
-		assert.Nil(ginkgo.GinkgoT(), err, "dns query error")
+		output, err := s.RunDigDNSClientFromK8s("@apisix-service-e2e-test", "-p", "9200", "github.com")
+		assert.Nil(ginkgo.GinkgoT(), err, "run dig error")
+		assert.Contains(ginkgo.GinkgoT(), output, "ADDITIONAL SECTION")
+
+		time.Sleep(3 * time.Second)
+		output = s.GetDeploymentLogs(scaffold.CoreDNSDeployment)
+		assert.Contains(ginkgo.GinkgoT(), output, "github.com. udp")
 	})
 })

test/e2e/suite-ingress/suite-ingress-resource/stream.go

@@ -16,8 +16,8 @@
 package ingress
 
 import (
-	"context"
 	"fmt"
+	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
@@ -62,55 +62,7 @@ spec:
 			resp.Body().Contains("x-my-value")
 		})
 		ginkgo.It("stream udp proxy", func() {
-			assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(`
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: coredns
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: coredns
-  template:
-    metadata:
-      labels:
-        app: coredns
-    spec:
-      containers:
-      - name: coredns
-        image: coredns/coredns:1.8.4
-        livenessProbe:
-          tcpSocket:
-            port: 53
-          initialDelaySeconds: 5
-          periodSeconds: 10
-        readinessProbe:
-          tcpSocket:
-            port: 53
-          initialDelaySeconds: 5
-          periodSeconds: 10
-        ports:    
-        - name: dns
-          containerPort: 53
-          protocol: UDP
-`))
-			assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(`
-kind: Service
-apiVersion: v1
-metadata:
-  name: coredns
-spec:
-  selector:
-    app: coredns
-  type: ClusterIP
-  ports:
-  - port: 53
-    targetPort: 53
-    protocol: UDP
-`))
-
-			s.EnsureNumEndpointsReady(ginkgo.GinkgoT(), "coredns", 1)
+			dnsSvc := s.NewCoreDNSService()
 
 			apisixRoute := fmt.Sprintf(`
 apiVersion: apisix.apache.org/v2
@@ -124,9 +76,9 @@ spec:
     match:
       ingressPort: 9200
     backend:
-      serviceName: coredns
-      servicePort: 53
-`)
+      serviceName: %s
+      servicePort: %d
+`, dnsSvc.Name, dnsSvc.Spec.Ports[0].Port)
 			assert.Nil(ginkgo.GinkgoT(), s.CreateVersionedApisixResource(apisixRoute))
 
 			err := s.EnsureNumApisixStreamRoutesCreated(1)
@@ -137,10 +89,13 @@ spec:
 			assert.Len(ginkgo.GinkgoT(), sr, 1)
 			assert.Equal(ginkgo.GinkgoT(), sr[0].ServerPort, int32(9200))
 			// test dns query
-			r := s.DNSResolver()
-			host := "httpbin.org"
-			_, err = r.LookupIPAddr(context.Background(), host)
-			assert.Nil(ginkgo.GinkgoT(), err, "dns query error")
+			output, err := s.RunDigDNSClientFromK8s("@apisix-service-e2e-test", "-p", "9200", "github.com")
+			assert.Nil(ginkgo.GinkgoT(), err, "run dig error")
+			assert.Contains(ginkgo.GinkgoT(), output, "ADDITIONAL SECTION")
+
+			time.Sleep(3 * time.Second)
+			output = s.GetDeploymentLogs(scaffold.CoreDNSDeployment)
+			assert.Contains(ginkgo.GinkgoT(), output, "github.com. udp")
 		})
 	}
 	ginkgo.Describe("suite-ingress-resource: scaffold v2", func() {

test/e2e/testdata/apisix-gw-config-v3.yaml

@@ -49,3 +49,8 @@ apisix:
 plugin_attr:
   prometheus:
     enable_export_server: false
+
+nginx_config:
+  worker_processes: 1
+  stream:
+    enable_access_log: true