From 10ebf3d1fa8fc66786fa22882447e29134b99615 Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Thu, 24 Jun 2021 10:38:37 +0800 Subject: [PATCH] [CI] Fix Windows conformance (#2286) * Pull necessary images for Windows Conformance tests * Added one e2e test to test NodePort Service on Windows to replace the one from Kubernetes conformance Signed-off-by: Zhecheng Li --- ci/jenkins/test.sh | 27 ++++++++++++---- test/e2e/fixtures.go | 6 ++++ test/e2e/service_test.go | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index eefc67edce9..25d4eba5615 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -31,7 +31,7 @@ DOCKER_REGISTRY=$(head -n1 "${WORKSPACE}/ci/docker-registry") IMAGE_PULL_POLICY="Always" WINDOWS_CONFORMANCE_FOCUS="\[sig-network\].+\[Conformance\]|\[sig-windows\]" -WINDOWS_CONFORMANCE_SKIP="\[LinuxOnly\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]|\[Privileged\]|should be able to change the type from|\[sig-network\] Services should be able to create a functioning NodePort service \[Conformance\]|Service endpoints latency should not be very high" +WINDOWS_CONFORMANCE_SKIP="\[LinuxOnly\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]|\[Privileged\]|should be able to change the type from|\[sig-network\] Services should be able to create a functioning NodePort service \[Conformance\]|Service endpoints latency should not be very high|should be able to create a functioning NodePort service for Windows" WINDOWS_NETWORKPOLICY_FOCUS="\[Feature:NetworkPolicy\]" WINDOWS_NETWORKPOLICY_SKIP="SKIP_NO_TESTCASE" CONFORMANCE_SKIP="\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]" @@ -235,21 +235,36 @@ function deliver_antrea_windows { cp -f build/yamls/*.yml $WORKDIR docker save -o antrea-ubuntu.tar projects.registry.vmware.com/antrea/antrea-ubuntu:latest - echo "===== Deliver Antrea to Linux nodes =====" + echo "===== Pull necessary images on Control-Plane node =====" + harbor_images=("agnhost:2.13" "nginx:1.15-alpine") + antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine") + for i in "${!harbor_images[@]}"; do + docker pull -q "${DOCKER_REGISTRY}/antrea/${harbor_images[i]}" + docker tag "${DOCKER_REGISTRY}/antrea/${harbor_images[i]}" "${antrea_images[i]}" + done + echo "===== Deliver Antrea to Linux worker nodes and pull necessary images on worker nodes =====" kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role && $1 !~ /win/ {print $6}' | while read IP; do rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" antrea-ubuntu.tar jenkins@${IP}:${WORKDIR}/antrea-ubuntu.tar ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true + + harbor_images=("agnhost:2.13" "nginx:1.15-alpine") + antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine") + for i in "${!harbor_images[@]}"; do + ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true + done done - echo "===== Deliver Antrea Windows to Windows nodes =====" + echo "===== Deliver Antrea Windows to Windows worker nodes and pull necessary images on Windows worker nodes =====" rm -f antrea-windows.tar.gz sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1 kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role && $1 ~ /win/ {print $6}' | while read IP; do govc snapshot.revert -vm.ip ${IP} win-initial - harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "e2eteam-agnhost:2.13" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2") - antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2") + # Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763 + # Use e2eteam/agnhost:2.13 instead + harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2") + antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2") for i in "${!harbor_images[@]}"; do - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true + ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true done # Use a script to run antrea agent in windows Network Policy cases diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 442dee74d30..29217af8045 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -99,6 +99,12 @@ func skipIfHasWindowsNodes(tb testing.TB) { } } +func skipIfNoWindowsNodes(tb testing.TB) { + if len(clusterInfo.windowsNodes) == 0 { + tb.Skipf("Skipping test as the cluster has Windows Nodes") + } +} + func ensureAntreaRunning(tb testing.TB, data *TestData) error { tb.Logf("Applying Antrea YAML") if err := data.deployAntrea(); err != nil { diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 786356ff783..0a3eb14bf3c 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -16,6 +16,7 @@ package e2e import ( "fmt" + "strings" "testing" "github.com/stretchr/testify/require" @@ -81,3 +82,68 @@ func (data *TestData) createClusterIPServiceAndBackendPods(t *testing.T, name st return svc, cleanup } + +// TestNodePortWindows tests NodePort Service on Windows Node. It is a temporary test to replace upstream Kubernetes one: +// https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/test/e2e/windows/service.go#L42 +// Issue: https://github.com/antrea-io/antrea/issues/2289 +func TestNodePortWindows(t *testing.T) { + skipIfNoWindowsNodes(t) + + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + svcName := "agnhost" + svcNode := nodeName(clusterInfo.windowsNodes[0]) + svc, cleanup := data.createAgnhostServiceAndBackendPods(t, svcName, svcNode, corev1.ServiceTypeNodePort) + defer cleanup() + t.Logf("%s Service is ready", svcName) + + // Unlike upstream Kubernetes Conformance, here the client is on a Linux Node (nodeName(0)). + // It doesn't need to be the control-plane for e2e test and other Linux workers will work as well. However, in this + // e2e framework, nodeName(0)/Control-plane Node is guaranteed to be a Linux one. + clientName := "agnhost-client" + require.NoError(t, data.createAgnhostPodOnNode(clientName, nodeName(0))) + defer data.deletePodAndWait(defaultTimeout, clientName) + _, err = data.podWaitForIPs(defaultTimeout, clientName, testNamespace) + require.NoError(t, err) + + nodeIP := clusterInfo.nodes[0].ip + nodePort := int(svc.Spec.Ports[0].NodePort) + addr := fmt.Sprintf("http://%s:%d", nodeIP, nodePort) + + cmd := append([]string{"curl", "--connect-timeout", "1", "--retry", "5", "--retry-connrefused"}, addr) + stdout, stderr, err := data.runCommandFromPod(testNamespace, clientName, agnhostContainerName, cmd) + if err != nil { + t.Errorf("Error when running command '%s' from Pod '%s', stdout: %s, stderr: %s, error: %v", + strings.Join(cmd, " "), clientName, stdout, stderr, err) + } else { + t.Logf("curl from Pod '%s' to '%s' succeeded", clientName, addr) + } +} + +func (data *TestData) createAgnhostServiceAndBackendPods(t *testing.T, name string, node string, svcType corev1.ServiceType) (*corev1.Service, func()) { + ipv4Protocol := corev1.IPv4Protocol + args := []string{"netexec", "--http-port=80", "--udp-port=80"} + require.NoError(t, data.createPodOnNode(name, node, agnhostImage, []string{}, args, nil, []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 80, + Protocol: corev1.ProtocolTCP, + }, + }, false, nil)) + _, err := data.podWaitForIPs(defaultTimeout, name, testNamespace) + require.NoError(t, err) + require.NoError(t, data.podWaitForRunning(defaultTimeout, name, testNamespace)) + svc, err := data.createService(name, 80, 80, map[string]string{"app": "agnhost"}, false, svcType, &ipv4Protocol) + require.NoError(t, err) + + cleanup := func() { + data.deletePodAndWait(defaultTimeout, name) + data.deleteServiceAndWait(defaultTimeout, name) + } + + return svc, cleanup +}