Skip to content

Commit

Permalink
[CI] Fix Windows conformance (#2286)
Browse files Browse the repository at this point in the history 
* Pull necessary images for Windows Conformance tests
* Added one e2e test to test NodePort Service on Windows to replace the
one from Kubernetes conformance

Signed-off-by: Zhecheng Li <lzhecheng@vmware.com>
  • Loading branch information
@lzhecheng
lzhecheng authored Jun 24, 2021
1 parent 6c350e0 commit 10ebf3d
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 6 deletions.
27 changes: 21 additions & 6 deletions ci/jenkins/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ DOCKER_REGISTRY=$(head -n1 "${WORKSPACE}/ci/docker-registry")
IMAGE_PULL_POLICY="Always"

WINDOWS_CONFORMANCE_FOCUS="\[sig-network\].+\[Conformance\]|\[sig-windows\]"
WINDOWS_CONFORMANCE_SKIP="\[LinuxOnly\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]|\[Privileged\]|should be able to change the type from|\[sig-network\] Services should be able to create a functioning NodePort service \[Conformance\]|Service endpoints latency should not be very high"
WINDOWS_CONFORMANCE_SKIP="\[LinuxOnly\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]|\[Privileged\]|should be able to change the type from|\[sig-network\] Services should be able to create a functioning NodePort service \[Conformance\]|Service endpoints latency should not be very high|should be able to create a functioning NodePort service for Windows"
WINDOWS_NETWORKPOLICY_FOCUS="\[Feature:NetworkPolicy\]"
WINDOWS_NETWORKPOLICY_SKIP="SKIP_NO_TESTCASE"
CONFORMANCE_SKIP="\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]"
Expand Down Expand Up @@ -235,21 +235,36 @@ function deliver_antrea_windows {
cp -f build/yamls/*.yml $WORKDIR
docker save -o antrea-ubuntu.tar projects.registry.vmware.com/antrea/antrea-ubuntu:latest

echo "===== Deliver Antrea to Linux nodes ====="
echo "===== Pull necessary images on Control-Plane node ====="
harbor_images=("agnhost:2.13" "nginx:1.15-alpine")
antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine")
for i in "${!harbor_images[@]}"; do
docker pull -q "${DOCKER_REGISTRY}/antrea/${harbor_images[i]}"
docker tag "${DOCKER_REGISTRY}/antrea/${harbor_images[i]}" "${antrea_images[i]}"
done
echo "===== Deliver Antrea to Linux worker nodes and pull necessary images on worker nodes ====="
kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role && $1 !~ /win/ {print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" antrea-ubuntu.tar jenkins@${IP}:${WORKDIR}/antrea-ubuntu.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true

harbor_images=("agnhost:2.13" "nginx:1.15-alpine")
antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine")
for i in "${!harbor_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
done
done

echo "===== Deliver Antrea Windows to Windows nodes ====="
echo "===== Deliver Antrea Windows to Windows worker nodes and pull necessary images on Windows worker nodes ====="
rm -f antrea-windows.tar.gz
sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1
kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role && $1 ~ /win/ {print $6}' | while read IP; do
govc snapshot.revert -vm.ip ${IP} win-initial
harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "e2eteam-agnhost:2.13" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2")
antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2")
# Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763
# Use e2eteam/agnhost:2.13 instead
harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2")
antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2")
for i in "${!harbor_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
done

# Use a script to run antrea agent in windows Network Policy cases
Expand Down
6 changes: 6 additions & 0 deletions test/e2e/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,12 @@ func skipIfHasWindowsNodes(tb testing.TB) {
}
}

func skipIfNoWindowsNodes(tb testing.TB) {
if len(clusterInfo.windowsNodes) == 0 {
tb.Skipf("Skipping test as the cluster has Windows Nodes")
}
}

func ensureAntreaRunning(tb testing.TB, data *TestData) error {
tb.Logf("Applying Antrea YAML")
if err := data.deployAntrea(); err != nil {
Expand Down
66 changes: 66 additions & 0 deletions test/e2e/service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package e2e

import (
"fmt"
"strings"
"testing"

"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -81,3 +82,68 @@ func (data *TestData) createClusterIPServiceAndBackendPods(t *testing.T, name st

return svc, cleanup
}

// TestNodePortWindows tests NodePort Service on Windows Node. It is a temporary test to replace upstream Kubernetes one:
// https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/test/e2e/windows/service.go#L42
// Issue: https://github.com/antrea-io/antrea/issues/2289
func TestNodePortWindows(t *testing.T) {
skipIfNoWindowsNodes(t)

data, err := setupTest(t)
if err != nil {
t.Fatalf("Error when setting up test: %v", err)
}
defer teardownTest(t, data)

svcName := "agnhost"
svcNode := nodeName(clusterInfo.windowsNodes[0])
svc, cleanup := data.createAgnhostServiceAndBackendPods(t, svcName, svcNode, corev1.ServiceTypeNodePort)
defer cleanup()
t.Logf("%s Service is ready", svcName)

// Unlike upstream Kubernetes Conformance, here the client is on a Linux Node (nodeName(0)).
// It doesn't need to be the control-plane for e2e test and other Linux workers will work as well. However, in this
// e2e framework, nodeName(0)/Control-plane Node is guaranteed to be a Linux one.
clientName := "agnhost-client"
require.NoError(t, data.createAgnhostPodOnNode(clientName, nodeName(0)))
defer data.deletePodAndWait(defaultTimeout, clientName)
_, err = data.podWaitForIPs(defaultTimeout, clientName, testNamespace)
require.NoError(t, err)

nodeIP := clusterInfo.nodes[0].ip
nodePort := int(svc.Spec.Ports[0].NodePort)
addr := fmt.Sprintf("http://%s:%d", nodeIP, nodePort)

cmd := append([]string{"curl", "--connect-timeout", "1", "--retry", "5", "--retry-connrefused"}, addr)
stdout, stderr, err := data.runCommandFromPod(testNamespace, clientName, agnhostContainerName, cmd)
if err != nil {
t.Errorf("Error when running command '%s' from Pod '%s', stdout: %s, stderr: %s, error: %v",
strings.Join(cmd, " "), clientName, stdout, stderr, err)
} else {
t.Logf("curl from Pod '%s' to '%s' succeeded", clientName, addr)
}
}

func (data *TestData) createAgnhostServiceAndBackendPods(t *testing.T, name string, node string, svcType corev1.ServiceType) (*corev1.Service, func()) {
ipv4Protocol := corev1.IPv4Protocol
args := []string{"netexec", "--http-port=80", "--udp-port=80"}
require.NoError(t, data.createPodOnNode(name, node, agnhostImage, []string{}, args, nil, []corev1.ContainerPort{
{
Name: "http",
ContainerPort: 80,
Protocol: corev1.ProtocolTCP,
},
}, false, nil))
_, err := data.podWaitForIPs(defaultTimeout, name, testNamespace)
require.NoError(t, err)
require.NoError(t, data.podWaitForRunning(defaultTimeout, name, testNamespace))
svc, err := data.createService(name, 80, 80, map[string]string{"app": "agnhost"}, false, svcType, &ipv4Protocol)
require.NoError(t, err)

cleanup := func() {
data.deletePodAndWait(defaultTimeout, name)
data.deleteServiceAndWait(defaultTimeout, name)
}

return svc, cleanup
}

0 comments on commit 10ebf3d

Please sign in to comment.