diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index f63c1554219..aa748ad33bf 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -238,9 +238,17 @@ func run(o *Options) error { } var egressController *egress.EgressController + var nodeIP net.IP + if nodeConfig.NodeIPv4Addr != nil { + nodeIP = nodeConfig.NodeIPv4Addr.IP + } else if nodeConfig.NodeIPv6Addr != nil { + nodeIP = nodeConfig.NodeIPv6Addr.IP + } else { + return fmt.Errorf("invalid NodeIPAddr in Node config: %v", nodeConfig) + } if features.DefaultFeatureGate.Enabled(features.Egress) { egressController, err = egress.NewEgressController( - ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP, + ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeIP, o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer, ) if err != nil { diff --git a/pkg/agent/memberlist/cluster.go b/pkg/agent/memberlist/cluster.go index 77cccfcc26d..90d0f6da7c7 100644 --- a/pkg/agent/memberlist/cluster.go +++ b/pkg/agent/memberlist/cluster.go @@ -257,13 +257,10 @@ func (c *Cluster) newClusterMember(node *corev1.Node) (string, error) { return "", fmt.Errorf("obtain IP addresses from K8s Node failed: %v", err) } nodeAddr := nodeAddrs.IPv4 - fmtStr := "%s:%d" if nodeAddr == nil { nodeAddr = nodeAddrs.IPv6 - fmtStr = "[%s]:%d" } - member := fmt.Sprintf(fmtStr, nodeAddr, c.bindPort) - return member, nil + return nodeAddr.String(), nil } func (c *Cluster) allClusterMembers() (clusterNodes []string, err error) { diff --git a/pkg/agent/util/ndp/doc.go b/pkg/agent/util/ndp/doc.go index 96177bca93e..b3beea27116 100644 --- a/pkg/agent/util/ndp/doc.go +++ b/pkg/agent/util/ndp/doc.go @@ -12,5 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ndp contains functions to send NDP advertisement on Linux. +// Package ndp contains functions to send NDP neighbor advertisement on Linux. package ndp diff --git a/test/e2e/egress_test.go b/test/e2e/egress_test.go index f908c1c4a6b..abd3cbff08b 100644 --- a/test/e2e/egress_test.go +++ b/test/e2e/egress_test.go @@ -25,9 +25,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + utilnet "k8s.io/utils/net" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/apis/crd/v1alpha2" @@ -35,8 +38,8 @@ import ( func TestEgress(t *testing.T) { skipIfProviderIs(t, "kind", "pkt_mark field is not properly supported for OVS userspace (netdev) datapath.") - // TODO: remove this after making the test support IPv6 and dual-stack. - skipIfIPv6Cluster(t) + skipIfHasWindowsNodes(t) + skipIfNumNodesLessThan(t, 2) data, err := setupTest(t) if err != nil { @@ -52,8 +55,9 @@ func TestEgress(t *testing.T) { ac := []configChange{ {"Egress", "true", true}, } + if err := data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { - t.Fatalf("Failed to enable NetworkPolicyStats feature: %v", err) + t.Fatalf("Failed to enable Egress feature: %v", err) } t.Run("testEgressClientIP", func(t *testing.T) { testEgressClientIP(t, data) }) @@ -64,163 +68,205 @@ func TestEgress(t *testing.T) { } func testEgressClientIP(t *testing.T, data *TestData) { - egressNode := controlPlaneNodeName() - egressNodeIP := controlPlaneNodeIP() - localIP0 := "1.1.1.10" - localIP1 := "1.1.1.11" - serverIP := "1.1.1.20" - fakeServer := "fakeserver" - - // Create a http server in another netns to fake an external server connected to the egress Node. - cmd := fmt.Sprintf(`ip netns add %[1]s && \ + tests := []struct { + name string + localIP0 string + localIP1 string + serverIP string + fakeServer string + ipMaskLen int + }{ + { + name: "ipv4-cluster", + localIP0: "1.1.1.10", + localIP1: "1.1.1.11", + serverIP: "1.1.1.20", + fakeServer: "eth-ipv4", + ipMaskLen: 24, + }, + { + name: "ipv6-cluster", + localIP0: "2021::aaa1", + localIP1: "2021::aaa2", + serverIP: "2021::aaa3", + fakeServer: "eth-ipv6", + ipMaskLen: 124, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + egressNode := controlPlaneNodeName() + var egressNodeIP string + if utilnet.IsIPv6String(tt.localIP0) { + skipIfNotIPv6Cluster(t) + egressNodeIP = controlPlaneNodeIPv6() + } else { + skipIfNotIPv4Cluster(t) + egressNodeIP = controlPlaneNodeIPv4() + } + + // Create a http server in another netns to fake an external server connected to the egress Node. + cmd := fmt.Sprintf(`ip netns add %[1]s && \ ip link add dev %[1]s-a type veth peer name %[1]s-b && \ ip link set dev %[1]s-a netns %[1]s && \ -ip addr add %[3]s/24 dev %[1]s-b && \ -ip addr add %[4]s/24 dev %[1]s-b && \ +ip addr add %[3]s/%[5]d dev %[1]s-b && \ +ip addr add %[4]s/%[5]d dev %[1]s-b && \ ip link set dev %[1]s-b up && \ -ip netns exec %[1]s ip addr add %[2]s/24 dev %[1]s-a && \ +ip netns exec %[1]s ip addr add %[2]s/%[5]d dev %[1]s-a && \ ip netns exec %[1]s ip link set dev %[1]s-a up && \ ip netns exec %[1]s ip route replace default via %[3]s && \ ip netns exec %[1]s /agnhost netexec -`, fakeServer, serverIP, localIP0, localIP1) - if err := data.createPodOnNode(fakeServer, testNamespace, egressNode, agnhostImage, []string{"sh", "-c", cmd}, nil, nil, nil, true, func(pod *v1.Pod) { - privileged := true - pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{Privileged: &privileged} - }); err != nil { - t.Fatalf("Failed to create server Pod: %v", err) - } - defer deletePodWrapper(t, data, fakeServer) - if err := data.podWaitForRunning(defaultTimeout, fakeServer, testNamespace); err != nil { - t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", fakeServer) - } - - localPod := "localpod" - remotePod := "remotepod" - if err := data.createBusyboxPodOnNode(localPod, testNamespace, egressNode); err != nil { - t.Fatalf("Failed to create local Pod: %v", err) - } - defer deletePodWrapper(t, data, localPod) - if err := data.podWaitForRunning(defaultTimeout, localPod, testNamespace); err != nil { - t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", localPod) - } - if err := data.createBusyboxPodOnNode(remotePod, testNamespace, workerNodeName(1)); err != nil { - t.Fatalf("Failed to create remote Pod: %v", err) - } - defer deletePodWrapper(t, data, remotePod) - if err := data.podWaitForRunning(defaultTimeout, remotePod, testNamespace); err != nil { - t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", remotePod) - } +`, tt.fakeServer, tt.serverIP, tt.localIP0, tt.localIP1, tt.ipMaskLen) + if err := data.createPodOnNode(tt.fakeServer, testNamespace, egressNode, agnhostImage, []string{"sh", "-c", cmd}, nil, nil, nil, true, func(pod *v1.Pod) { + privileged := true + pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{Privileged: &privileged} + }); err != nil { + t.Fatalf("Failed to create server Pod: %v", err) + } + defer deletePodWrapper(t, data, tt.fakeServer) + if err := data.podWaitForRunning(defaultTimeout, tt.fakeServer, testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", tt.fakeServer) + } - // getClientIP gets the translated client IP by accessing the API that replies the request's client IP. - getClientIP := func(pod string) (string, string, error) { - cmd := []string{"wget", "-T", "3", "-O", "-", fmt.Sprintf("%s:8080/clientip", serverIP)} - return data.runCommandFromPod(testNamespace, pod, busyboxContainerName, cmd) - } + localPod := fmt.Sprintf("localpod%s", tt.name) + remotePod := fmt.Sprintf("remotepod%s", tt.name) + if err := data.createBusyboxPodOnNode(localPod, testNamespace, egressNode); err != nil { + t.Fatalf("Failed to create local Pod: %v", err) + } + defer deletePodWrapper(t, data, localPod) + if err := data.podWaitForRunning(defaultTimeout, localPod, testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", localPod) + } + if err := data.createBusyboxPodOnNode(remotePod, testNamespace, workerNodeName(1)); err != nil { + t.Fatalf("Failed to create remote Pod: %v", err) + } + defer deletePodWrapper(t, data, remotePod) + if err := data.podWaitForRunning(defaultTimeout, remotePod, testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", remotePod) + } - // assertClientIP asserts the Pod is translated to the provided client IP. - assertClientIP := func(pod string, clientIP string) { - var exeErr error - var stdout, stderr string - if err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) { - stdout, stderr, exeErr = getClientIP(pod) - if exeErr != nil { - return false, nil + // getClientIP gets the translated client IP by accessing the API that replies the request's client IP. + getClientIP := func(pod string) (string, string, error) { + serverIPStr := tt.serverIP + if utilnet.IsIPv6String(tt.localIP0) { + serverIPStr = fmt.Sprintf("[%s]", tt.serverIP) + } + cmd := []string{"wget", "-T", "3", "-O", "-", fmt.Sprintf("%s:8080/clientip", serverIPStr)} + return data.runCommandFromPod(testNamespace, pod, busyboxContainerName, cmd) } - // The stdout return is in this format: x.x.x.x:port or [xx:xx:xx::x]:port - host, _, err := net.SplitHostPort(stdout) - if err != nil { - return false, nil + + // assertClientIP asserts the Pod is translated to the provided client IP. + assertClientIP := func(pod string, clientIPs ...string) { + var exeErr error + var stdout, stderr string + if err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) { + stdout, stderr, exeErr = getClientIP(pod) + if exeErr != nil { + return false, nil + } + + // The stdout return is in this format: x.x.x.x:port or [xx:xx:xx::x]:port + host, _, err := net.SplitHostPort(stdout) + if err != nil { + return false, nil + } + for _, cip := range clientIPs { + if cip == host { + return true, nil + } + } + return false, nil + }); err != nil { + t.Fatalf("Failed to get expected client IPs %s for Pod %s, stdout: %s, stderr: %s, err: %v", clientIPs, pod, stdout, stderr, exeErr) + } } - return host == clientIP, nil - }); err != nil { - t.Fatalf("Failed to get expected client IP %s for Pod %s, stdout: %s, stderr: %s, err: %v", clientIP, pod, stdout, stderr, exeErr) - } - } - // assertConnError asserts the Pod is not able to access the API that replies the request's client IP. - assertConnError := func(pod string) { - var exeErr error - var stdout, stderr string - if err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) { - stdout, stderr, exeErr = getClientIP(pod) - if exeErr != nil { - return true, nil + // assertConnError asserts the Pod is not able to access the API that replies the request's client IP. + assertConnError := func(pod string) { + var exeErr error + var stdout, stderr string + if err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) { + stdout, stderr, exeErr = getClientIP(pod) + if exeErr != nil { + return true, nil + } + return false, nil + }); err != nil { + t.Fatalf("Failed to get expected error, stdout: %v, stderr: %v, err: %v", stdout, stderr, exeErr) + } } - return false, nil - }); err != nil { - t.Fatalf("Failed to get expected error, stdout: %v, stderr: %v, err: %v", stdout, stderr, exeErr) - } - } - // As the fake server runs in a netns of the Egress Node, only egress Node can reach the server, Pods running on - // other Nodes cannot reach it before Egress is added. - assertClientIP(localPod, localIP0) - assertConnError(remotePod) + // As the fake server runs in a netns of the Egress Node, only egress Node can reach the server, Pods running on + // other Nodes cannot reach it before Egress is added. + assertClientIP(localPod, tt.localIP0, tt.localIP1) + assertConnError(remotePod) - t.Logf("Creating an Egress applying to both Pods") - matchExpressions := []metav1.LabelSelectorRequirement{ - { - Key: "antrea-e2e", - Operator: metav1.LabelSelectorOpExists, - }, - } - egress := data.createEgress(t, "egress-", matchExpressions, nil, "", egressNodeIP) - defer data.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) - assertClientIP(localPod, egressNodeIP) - assertClientIP(remotePod, egressNodeIP) + t.Logf("Creating an Egress applying to both Pods") + matchExpressions := []metav1.LabelSelectorRequirement{ + { + Key: "antrea-e2e", + Operator: metav1.LabelSelectorOpExists, + }, + } + egress := data.createEgress(t, "egress-", matchExpressions, nil, "", egressNodeIP) + defer data.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) + assertClientIP(localPod, egressNodeIP) + assertClientIP(remotePod, egressNodeIP) - var err error - err = wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) { - egress, err = data.crdClient.CrdV1alpha2().Egresses().Get(context.TODO(), egress.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return egress.Status.EgressNode == egressNode, nil - }) - assert.NoError(t, err, "Egress failed to reach expected status") + var err error + err = wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) { + egress, err = data.crdClient.CrdV1alpha2().Egresses().Get(context.TODO(), egress.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return egress.Status.EgressNode == egressNode, nil + }) + assert.NoError(t, err, "Egress failed to reach expected status") - t.Log("Updating the Egress's AppliedTo to remotePod only") - egress.Spec.AppliedTo = v1alpha2.AppliedTo{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"antrea-e2e": remotePod}, - }, - } - egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Failed to update Egress %v: %v", egress, err) - } - assertClientIP(localPod, localIP0) - assertClientIP(remotePod, egressNodeIP) + t.Log("Updating the Egress's AppliedTo to remotePod only") + egress.Spec.AppliedTo = v1alpha2.AppliedTo{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"antrea-e2e": remotePod}, + }, + } + egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update Egress %v: %v", egress, err) + } + assertClientIP(localPod, tt.localIP0, tt.localIP1) + assertClientIP(remotePod, egressNodeIP) - t.Log("Updating the Egress's AppliedTo to localPod only") - egress.Spec.AppliedTo = v1alpha2.AppliedTo{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"antrea-e2e": localPod}, - }, - } - egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Failed to update Egress %v: %v", egress, err) - } - assertClientIP(localPod, egressNodeIP) - assertConnError(remotePod) + t.Log("Updating the Egress's AppliedTo to localPod only") + egress.Spec.AppliedTo = v1alpha2.AppliedTo{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"antrea-e2e": localPod}, + }, + } + egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update Egress %v: %v", egress, err) + } + assertClientIP(localPod, egressNodeIP) + assertConnError(remotePod) - t.Logf("Updating the Egress's EgressIP to %s", localIP1) - egress.Spec.EgressIP = localIP1 - egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Failed to update Egress %v: %v", egress, err) - } - assertClientIP(localPod, localIP1) - assertConnError(remotePod) + t.Logf("Updating the Egress's EgressIP to %s", tt.localIP1) + egress.Spec.EgressIP = tt.localIP1 + egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update Egress %v: %v", egress, err) + } + assertClientIP(localPod, tt.localIP1) + assertConnError(remotePod) - t.Log("Deleting the Egress") - err = data.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Failed to delete Egress %v: %v", egress, err) + t.Log("Deleting the Egress") + err = data.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete Egress %v: %v", egress, err) + } + assertClientIP(localPod, tt.localIP0, tt.localIP1) + assertConnError(remotePod) + }) } - assertClientIP(localPod, localIP0) - assertConnError(remotePod) } func testEgressCRUD(t *testing.T, data *TestData) { @@ -244,6 +290,18 @@ func testEgressCRUD(t *testing.T, data *TestData) { expectedNodes: sets.NewString(nodeName(0)), expectedTotal: 2, }, + { + name: "single matching Node with IPv6 range", + ipRange: v1alpha2.IPRange{CIDR: "2021::aaa0/124"}, + nodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + v1.LabelHostname: nodeName(0), + }, + }, + expectedEgressIP: "2021::aaa1", + expectedNodes: sets.NewString(nodeName(0)), + expectedTotal: 15, + }, { name: "two matching Nodes", ipRange: v1alpha2.IPRange{Start: "169.254.101.10", End: "169.254.101.11"}, @@ -275,6 +333,11 @@ func testEgressCRUD(t *testing.T, data *TestData) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if utilnet.IsIPv6String(tt.expectedEgressIP) { + skipIfNotIPv6Cluster(t) + } else { + skipIfNotIPv4Cluster(t) + } pool := data.createExternalIPPool(t, "pool-", tt.ipRange, tt.nodeSelector.MatchExpressions, tt.nodeSelector.MatchLabels) defer data.crdClient.CrdV1alpha2().ExternalIPPools().Delete(context.TODO(), pool.Name, metav1.DeleteOptions{}) @@ -334,43 +397,72 @@ func testEgressUpdateEgressIP(t *testing.T, data *TestData) { name string originalNode string newNode string + originalIPRange v1alpha2.IPRange originalEgressIP string + newIPRange v1alpha2.IPRange newEgressIP string }{ { - name: "same Node", - originalNode: nodeName(0), - newNode: nodeName(0), + name: "same Node", + originalNode: nodeName(0), + newNode: nodeName(0), + originalIPRange: v1alpha2.IPRange{CIDR: "169.254.100.0/30"}, + originalEgressIP: "169.254.100.1", + newIPRange: v1alpha2.IPRange{CIDR: "169.254.101.0/30"}, + newEgressIP: "169.254.101.1", }, { - name: "different Nodes", - originalNode: nodeName(0), - newNode: nodeName(1), + name: "different Nodes", + originalNode: nodeName(0), + newNode: nodeName(1), + originalIPRange: v1alpha2.IPRange{CIDR: "169.254.100.0/30"}, + originalEgressIP: "169.254.100.1", + newIPRange: v1alpha2.IPRange{CIDR: "169.254.101.0/30"}, + newEgressIP: "169.254.101.1", + }, + { + name: "different Nodes in IPv6 cluster", + originalNode: nodeName(0), + newNode: nodeName(1), + originalIPRange: v1alpha2.IPRange{CIDR: "2021::aaa0/124"}, + originalEgressIP: "2021::aaa1", + newIPRange: v1alpha2.IPRange{CIDR: "2021::bbb0/124"}, + newEgressIP: "2021::bbb1", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - originalPool := data.createExternalIPPool(t, "originalpool-", v1alpha2.IPRange{CIDR: "169.254.100.0/30"}, nil, map[string]string{v1.LabelHostname: tt.originalNode}) + if utilnet.IsIPv6String(tt.originalEgressIP) { + skipIfNotIPv6Cluster(t) + } else { + skipIfNotIPv4Cluster(t) + } + originalPool := data.createExternalIPPool(t, "originalpool-", tt.originalIPRange, nil, map[string]string{v1.LabelHostname: tt.originalNode}) defer data.crdClient.CrdV1alpha2().ExternalIPPools().Delete(context.TODO(), originalPool.Name, metav1.DeleteOptions{}) - newPool := data.createExternalIPPool(t, "newpool-", v1alpha2.IPRange{CIDR: "169.254.101.0/30"}, nil, map[string]string{v1.LabelHostname: tt.newNode}) + newPool := data.createExternalIPPool(t, "newpool-", tt.newIPRange, nil, map[string]string{v1.LabelHostname: tt.newNode}) defer data.crdClient.CrdV1alpha2().ExternalIPPools().Delete(context.TODO(), newPool.Name, metav1.DeleteOptions{}) - originalIP := "169.254.100.1" - newIP := "169.254.101.1" egress := data.createEgress(t, "egress-", nil, map[string]string{"foo": "bar"}, originalPool.Name, "") defer data.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) - egress, err := data.checkEgressState(egress.Name, originalIP, tt.originalNode, "", time.Second) + egress, err := data.checkEgressState(egress.Name, tt.originalEgressIP, tt.originalNode, "", time.Second) require.NoError(t, err) + // The Egress maybe has been modified. toUpdate := egress.DeepCopy() - toUpdate.Spec.ExternalIPPool = newPool.Name - toUpdate.Spec.EgressIP = newIP - egress, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), toUpdate, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + toUpdate.Spec.ExternalIPPool = newPool.Name + toUpdate.Spec.EgressIP = tt.newEgressIP + _, err = data.crdClient.CrdV1alpha2().Egresses().Update(context.TODO(), toUpdate, metav1.UpdateOptions{}) + if err != nil && errors.IsConflict(err) { + toUpdate, _ = data.crdClient.CrdV1alpha2().Egresses().Get(context.TODO(), egress.Name, metav1.GetOptions{}) + } + return err + }) require.NoError(t, err, "Failed to update Egress") - _, err = data.checkEgressState(egress.Name, newIP, tt.newNode, "", time.Second) + _, err = data.checkEgressState(egress.Name, tt.newEgressIP, tt.newNode, "", time.Second) require.NoError(t, err) - exists, err := hasIP(data, tt.originalNode, originalIP) + exists, err := hasIP(data, tt.originalNode, tt.originalEgressIP) require.NoError(t, err, "Failed to check if IP exists on Node") assert.False(t, exists, "Found stale IP on Node") }) @@ -378,55 +470,107 @@ func testEgressUpdateEgressIP(t *testing.T, data *TestData) { } func testEgressUpdateNodeSelector(t *testing.T, data *TestData) { - updateNodeSelector := func(poolName, evictNode string, ensureExists bool) { - pool, err := data.crdClient.CrdV1alpha2().ExternalIPPools().Get(context.TODO(), poolName, metav1.GetOptions{}) - require.NoError(t, err, "Failed to get ExternalIPPool %v", pool) - newNodes := sets.NewString(pool.Spec.NodeSelector.MatchExpressions[0].Values...) - if ensureExists { - newNodes.Insert(evictNode) - } else { - newNodes.Delete(evictNode) - } - pool.Spec.NodeSelector.MatchExpressions[0].Values = newNodes.List() - _, err = data.crdClient.CrdV1alpha2().ExternalIPPools().Update(context.TODO(), pool, metav1.UpdateOptions{}) - require.NoError(t, err, "Failed to update ExternalIPPool %v", pool) - } - shrinkEgressNodes := func(poolName, evictNode string) { - // Remove one Node from the node candidates. - updateNodeSelector(poolName, evictNode, false) + tests := []struct { + name string + ipRange v1alpha2.IPRange + ipVersion int + }{ + { + name: "IPv4 cluster", + ipRange: v1alpha2.IPRange{CIDR: "169.254.100.0/30"}, + ipVersion: 4, + }, + { + name: "IPv6 cluster", + ipRange: v1alpha2.IPRange{CIDR: "2021::aaa1/124"}, + ipVersion: 6, + }, } - restoreEgressNodes := func(poolName, evictNode string) { - // Add the removed Node back to the node candidates. - updateNodeSelector(poolName, evictNode, true) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + switch tt.ipVersion { + case 4: + skipIfNotIPv4Cluster(t) + case 6: + skipIfNotIPv6Cluster(t) + } + updateNodeSelector := func(poolName, evictNode string, ensureExists bool) { + pool, err := data.crdClient.CrdV1alpha2().ExternalIPPools().Get(context.TODO(), poolName, metav1.GetOptions{}) + require.NoError(t, err, "Failed to get ExternalIPPool %v", pool) + newNodes := sets.NewString(pool.Spec.NodeSelector.MatchExpressions[0].Values...) + if ensureExists { + newNodes.Insert(evictNode) + } else { + newNodes.Delete(evictNode) + } + pool.Spec.NodeSelector.MatchExpressions[0].Values = newNodes.List() + _, err = data.crdClient.CrdV1alpha2().ExternalIPPools().Update(context.TODO(), pool, metav1.UpdateOptions{}) + require.NoError(t, err, "Failed to update ExternalIPPool %v", pool) + } + shrinkEgressNodes := func(poolName, evictNode string) { + // Remove one Node from the node candidates. + updateNodeSelector(poolName, evictNode, false) + } + restoreEgressNodes := func(poolName, evictNode string) { + // Add the removed Node back to the node candidates. + updateNodeSelector(poolName, evictNode, true) + } + // Egress IP migration should happen fast when it's caused by nodeSelector update. + // No IP should be left on the evicted Node. + testEgressMigration(t, data, shrinkEgressNodes, restoreEgressNodes, true, time.Second, &tt.ipRange) + }) } - // Egress IP migration should happen fast when it's caused by nodeSelector update. - // No IP should be left on the evicted Node. - testEgressMigration(t, data, shrinkEgressNodes, restoreEgressNodes, true, time.Second) } func testEgressNodeFailure(t *testing.T, data *TestData) { - signalAgent := func(nodeName, signal string) { - cmd := fmt.Sprintf("pkill -%s antrea-agent", signal) - rc, stdout, stderr, err := RunCommandOnNode(nodeName, cmd) - if rc != 0 || err != nil { - t.Errorf("Error when running command '%s' on Node '%s', rc: %d, stdout: %s, stderr: %s, error: %v", - cmd, nodeName, rc, stdout, stderr, err) - } - } - pauseAgent := func(_, evictNode string) { - // Send "STOP" signal to antrea-agent. - signalAgent(evictNode, "STOP") + tests := []struct { + name string + ipRange v1alpha2.IPRange + ipVersion int + }{ + { + name: "IPv4 cluster", + ipRange: v1alpha2.IPRange{CIDR: "169.254.100.0/30"}, + ipVersion: 4, + }, + { + name: "IPv6 cluster", + ipRange: v1alpha2.IPRange{CIDR: "2021::aaa1/124"}, + ipVersion: 6, + }, } - restoreAgent := func(_, evictNode string) { - // Send "CONT" signal to antrea-agent. - signalAgent(evictNode, "CONT") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + switch tt.ipVersion { + case 4: + skipIfNotIPv4Cluster(t) + case 6: + skipIfNotIPv6Cluster(t) + } + signalAgent := func(nodeName, signal string) { + cmd := fmt.Sprintf("pkill -%s antrea-agent", signal) + rc, stdout, stderr, err := RunCommandOnNode(nodeName, cmd) + if rc != 0 || err != nil { + t.Errorf("Error when running command '%s' on Node '%s', rc: %d, stdout: %s, stderr: %s, error: %v", + cmd, nodeName, rc, stdout, stderr, err) + } + } + pauseAgent := func(_, evictNode string) { + // Send "STOP" signal to antrea-agent. + signalAgent(evictNode, "STOP") + } + restoreAgent := func(_, evictNode string) { + // Send "CONT" signal to antrea-agent. + signalAgent(evictNode, "CONT") + } + // Egress IP migration may take a few seconds when it's caused by Node failure detection. + // Skip checking Egress IP on the evicted Node because Egress IP will be left on it (no running antrea-agent). + testEgressMigration(t, data, pauseAgent, restoreAgent, false, 3*time.Second, &tt.ipRange) + }) } - // Egress IP migration may take a few seconds when it's caused by Node failure detection. - // Skip checking Egress IP on the evicted Node because Egress IP will be left on it (no running antrea-agent). - testEgressMigration(t, data, pauseAgent, restoreAgent, false, 3*time.Second) } -func testEgressMigration(t *testing.T, data *TestData, triggerFunc, revertFunc func(poolName, evictNode string), checkEvictNode bool, timeout time.Duration) { +func testEgressMigration(t *testing.T, data *TestData, triggerFunc, revertFunc func(poolName, evictNode string), checkEvictNode bool, timeout time.Duration, ipRange *v1alpha2.IPRange) { nodeCandidates := sets.NewString(nodeName(0), nodeName(1)) matchExpressions := []metav1.LabelSelectorRequirement{ { @@ -435,7 +579,7 @@ func testEgressMigration(t *testing.T, data *TestData, triggerFunc, revertFunc f Values: nodeCandidates.List(), }, } - externalIPPoolTwoNodes := data.createExternalIPPool(t, "pool-", v1alpha2.IPRange{CIDR: "169.254.100.0/30"}, matchExpressions, nil) + externalIPPoolTwoNodes := data.createExternalIPPool(t, "pool-", *ipRange, matchExpressions, nil) defer data.crdClient.CrdV1alpha2().ExternalIPPools().Delete(context.TODO(), externalIPPoolTwoNodes.Name, metav1.DeleteOptions{}) egress := data.createEgress(t, "egress-", nil, map[string]string{"foo": "bar"}, externalIPPoolTwoNodes.Name, "") @@ -513,7 +657,7 @@ func hasIP(data *TestData, nodeName string, ip string) (bool, error) { if err != nil { return false, err } - return strings.Contains(stdout, ip+"/32"), nil + return strings.Contains(stdout, ip+"/32") || strings.Contains(stdout, ip+"/128"), nil } func (data *TestData) createExternalIPPool(t *testing.T, generateName string, ipRange v1alpha2.IPRange, matchExpressions []metav1.LabelSelectorRequirement, matchLabels map[string]string) *v1alpha2.ExternalIPPool { diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 0dbb1373010..6203babb2cb 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -76,7 +76,7 @@ func skipIfIPv6Cluster(tb testing.TB) { func skipIfNotIPv6Cluster(tb testing.TB) { if clusterInfo.podV6NetworkCIDR == "" { - tb.Skipf("Skipping test as it is not needed in IPv4 cluster") + tb.Skipf("Skipping test as it requires IPv6 addresses but the IPv6 network CIDR is not set") } } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index ca017e849ad..2e60c4a7a14 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -47,6 +47,7 @@ import ( "k8s.io/client-go/tools/remotecommand" "k8s.io/component-base/featuregate" aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + utilnet "k8s.io/utils/net" "antrea.io/antrea/pkg/agent/config" crdclientset "antrea.io/antrea/pkg/client/clientset/versioned" @@ -120,7 +121,8 @@ const ( type ClusterNode struct { idx int // 0 for control-plane Node name string - ip string + ipv4Addr string + ipv6Addr string podV4NetworkCIDR string podV6NetworkCIDR string gwV4Addr string @@ -128,6 +130,13 @@ type ClusterNode struct { os string } +func (n ClusterNode) ip() string { + if n.ipv4Addr != "" { + return n.ipv4Addr + } + return n.ipv6Addr +} + type ClusterInfo struct { numNodes int podV4NetworkCIDR string @@ -135,7 +144,8 @@ type ClusterInfo struct { svcV4NetworkCIDR string svcV6NetworkCIDR string controlPlaneNodeName string - controlPlaneNodeIP string + controlPlaneNodeIPv4 string + controlPlaneNodeIPv6 string nodes map[int]ClusterNode nodesOS map[string]string windowsNodes []int @@ -276,7 +286,7 @@ func workerNodeIP(idx int) string { if !ok { return "" } - return node.ip + return node.ip() } // nodeGatewayIPs returns the Antrea gateway's IPv4 address and IPv6 address for the provided Node @@ -293,8 +303,12 @@ func controlPlaneNodeName() string { return clusterInfo.controlPlaneNodeName } -func controlPlaneNodeIP() string { - return clusterInfo.controlPlaneNodeIP +func controlPlaneNodeIPv4() string { + return clusterInfo.controlPlaneNodeIPv4 +} + +func controlPlaneNodeIPv6() string { + return clusterInfo.controlPlaneNodeIPv6 } // nodeName returns an empty string if there is no Node with the provided idx. If idx is 0, the name @@ -314,7 +328,7 @@ func nodeIP(idx int) string { if !ok { return "" } - return node.ip + return node.ip() } func labelNodeRoleControlPlane() string { @@ -376,11 +390,15 @@ func collectClusterInfo() error { return ok }() - var nodeIP string + var nodeIPv4 string + var nodeIPv6 string for _, address := range node.Status.Addresses { if address.Type == corev1.NodeInternalIP { - nodeIP = address.Address - break + if utilnet.IsIPv6String(address.Address) { + nodeIPv6 = address.Address + } else if utilnet.IsIPv4String(address.Address) { + nodeIPv4 = address.Address + } } } @@ -389,7 +407,8 @@ func collectClusterInfo() error { if isControlPlaneNode { nodeIdx = 0 clusterInfo.controlPlaneNodeName = node.Name - clusterInfo.controlPlaneNodeIP = nodeIP + clusterInfo.controlPlaneNodeIPv4 = nodeIPv4 + clusterInfo.controlPlaneNodeIPv6 = nodeIPv6 } else { nodeIdx = workerIdx workerIdx++ @@ -426,7 +445,8 @@ func collectClusterInfo() error { clusterInfo.nodes[nodeIdx] = ClusterNode{ idx: nodeIdx, name: node.Name, - ip: nodeIP, + ipv4Addr: nodeIPv4, + ipv6Addr: nodeIPv6, podV4NetworkCIDR: podV4NetworkCIDR, podV6NetworkCIDR: podV6NetworkCIDR, gwV4Addr: gwV4Addr, diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 3d698c4ab11..d62612ffaf4 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -133,7 +133,7 @@ func TestNodePortWindows(t *testing.T) { _, err = data.podWaitForIPs(defaultTimeout, clientName, testNamespace) require.NoError(t, err) - nodeIP := clusterInfo.nodes[0].ip + nodeIP := clusterInfo.nodes[0].ip() nodePort := int(svc.Spec.Ports[0].NodePort) addr := fmt.Sprintf("http://%s:%d", nodeIP, nodePort) diff --git a/test/e2e/wireguard_test.go b/test/e2e/wireguard_test.go index 205ba4586bc..0334b55ac0c 100644 --- a/test/e2e/wireguard_test.go +++ b/test/e2e/wireguard_test.go @@ -116,7 +116,7 @@ func testWireGuardTunnelConnectivity(t *testing.T, data *TestData) { var nodeIP string for _, n := range clusterInfo.nodes { if n.name == nodeName1 { - nodeIP = n.ip + nodeIP = n.ip() break } }