[IPv6] Implement L3 connectivity for IPv6 traffic (antrea-io#1011)

1. Use IPv6 in iptables and ipset configuration.
2. Identifiy IPv6 address and configure in OpenFlow.
3. Use Node Internal address for tunnel.

Makefile

@@ -128,6 +128,7 @@ docker-test-integration: .coverage
 		-v $(DOCKER_CACHE)/gocache:/tmp/gocache \
 		-v $(CURDIR)/.coverage:/usr/src/github.com/vmware-tanzu/antrea/.coverage \
 		-v $(CURDIR):/usr/src/github.com/vmware-tanzu/antrea:ro \
+		-v /lib/modules:/lib/modules \
 		antrea/test test-integration $(USERID) $(GRPID)
 
 .PHONY: docker-tidy

pkg/agent/controller/networkpolicy/reconciler_test.go

@@ -660,7 +660,7 @@ func TestReconcilerUpdate(t *testing.T) {
 	ifaceStore.AddInterface(
 		&interfacestore.InterfaceConfig{
 			InterfaceName:            util.GenerateContainerInterfaceName("pod3", "ns1", "container3"),
-			IP:                       net.ParseIP("4.4.4.4"),
+			IPs:                      []net.IP{net.ParseIP("4.4.4.4")},
 			ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod3", PodNamespace: "ns1", ContainerID: "container3"},
 			OVSPortConfig:            &interfacestore.OVSPortConfig{OFPort: 3}})
 	tests := []struct {

pkg/agent/controller/noderoute/node_route_controller.go

@@ -157,12 +157,11 @@ func (c *Controller) removeStaleGatewayRoutes() error {
 	// running, so the route to local Pods will be desired as well.
 	var desiredPodCIDRs, desiredRemoteNodeIPs []string
 	for _, node := range nodes {
-		// PodCIDR is allocated by K8s NodeIpamController asynchronously so it's possible we see a Node
-		// with no PodCIDR set when it just joins the cluster.
-		if node.Spec.PodCIDR == "" {
+		podCIDRs := getPodCIDRsOnNode(node)
+		if len(podCIDRs) == 0 {
 			continue
 		}
-		desiredPodCIDRs = append(desiredPodCIDRs, node.Spec.PodCIDR)
+		desiredPodCIDRs = append(desiredPodCIDRs, podCIDRs...)
 
 		if node.Name == c.nodeConfig.Name {
 			continue
@@ -370,16 +369,18 @@ func (c *Controller) syncNodeRoute(nodeName string) error {
 func (c *Controller) deleteNodeRoute(nodeName string) error {
 	klog.Infof("Deleting routes and flows to Node %s", nodeName)
 
-	obj, installed := c.installedNodes.Load(nodeName)
+	objs, installed := c.installedNodes.Load(nodeName)
 	if !installed {
 		// Route is not added for this Node.
 		return nil
 	}
-	nodeRouteInfo := obj.(*nodeRouteInfo)
-	if err := c.routeClient.DeleteRoutes(nodeRouteInfo.podCIDR, nodeRouteInfo.nodeIP); err != nil {
-		return fmt.Errorf("failed to delete the route to Node %s: %v", nodeName, err)
-	}
 
+	nodeRouteInfos := objs.([]*nodeRouteInfo)
+	for _, nodeRouteInfo := range nodeRouteInfos {
+		if err := c.routeClient.DeleteRoutes(nodeRouteInfo.podCIDR, nodeRouteInfo.nodeIP); err != nil {
+			return fmt.Errorf("failed to delete the route to Node %s: %v", nodeName, err)
+		}
+	}
 	if err := c.ofClient.UninstallNodeFlows(nodeName); err != nil {
 		return fmt.Errorf("failed to uninstall flows to Node %s: %v", nodeName, err)
 	}
@@ -407,25 +408,20 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
 		return nil
 	}
 
-	klog.Infof("Adding routes and flows to Node %s, podCIDR: %s, addresses: %v",
-		nodeName, node.Spec.PodCIDR, node.Status.Addresses)
-
-	if node.Spec.PodCIDR == "" {
-		klog.Errorf("PodCIDR is empty for Node %s", nodeName)
-		// Does not help to return an error and trigger controller retries.
-		return nil
-	}
-	peerPodCIDRAddr, peerPodCIDR, err := net.ParseCIDR(node.Spec.PodCIDR)
-	if err != nil {
-		klog.Errorf("Failed to parse PodCIDR %s for Node %s", node.Spec.PodCIDR, nodeName)
+	podCIDRStrs := getPodCIDRsOnNode(node)
+	if len(podCIDRStrs) == 0 {
+		// If no valid PodCIDR is configured in Node.Spec, return immediately.
 		return nil
 	}
+
+	klog.Infof("Adding routes and flows to Node %s, podCIDRs: %v, addresses: %v",
+		nodeName, podCIDRStrs, node.Status.Addresses)
+
 	peerNodeIP, err := GetNodeAddr(node)
 	if err != nil {
 		klog.Errorf("Failed to retrieve IP address of Node %s: %v", nodeName, err)
 		return nil
 	}
-	peerGatewayIP := ip.NextIP(peerPodCIDRAddr)
 
 	ipsecTunOFPort := int32(0)
 	if c.networkConfig.EnableIPSecTunnel {
@@ -437,29 +433,52 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
 		}
 	}
 
-	err = c.ofClient.InstallNodeFlows(
-		nodeName,
-		c.nodeConfig.GatewayConfig.MAC,
-		*peerPodCIDR,
-		peerGatewayIP,
-		peerNodeIP,
-		config.DefaultTunOFPort,
-		uint32(ipsecTunOFPort))
-	if err != nil {
-		return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err)
-	}
+	var nodeRouteInfos []*nodeRouteInfo
+	for _, podCIDR := range podCIDRStrs {
+		peerPodCIDRAddr, peerPodCIDR, err := net.ParseCIDR(podCIDR)
+		if err != nil {
+			klog.Errorf("Failed to parse PodCIDR %s for Node %s", podCIDR, nodeName)
+			return nil
+		}
+		peerGatewayIP := ip.NextIP(peerPodCIDRAddr)
+		err = c.ofClient.InstallNodeFlows(
+			nodeName,
+			c.nodeConfig.GatewayConfig.MAC,
+			*peerPodCIDR,
+			peerGatewayIP,
+			peerNodeIP,
+			config.DefaultTunOFPort,
+			uint32(ipsecTunOFPort))
+		if err != nil {
+			return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err)
+		}
 
-	if err := c.routeClient.AddRoutes(peerPodCIDR, peerNodeIP, peerGatewayIP); err != nil {
-		return err
+		if err := c.routeClient.AddRoutes(peerPodCIDR, peerNodeIP, peerGatewayIP); err != nil {
+			return err
+		}
+		nodeRouteInfos = append(nodeRouteInfos, &nodeRouteInfo{
+			podCIDR:   peerPodCIDR,
+			nodeIP:    peerNodeIP,
+			gatewayIP: peerGatewayIP,
+		})
 	}
-	c.installedNodes.Store(nodeName, &nodeRouteInfo{
-		podCIDR:   peerPodCIDR,
-		nodeIP:    peerNodeIP,
-		gatewayIP: peerGatewayIP,
-	})
+	c.installedNodes.Store(nodeName, nodeRouteInfos)
 	return err
 }
 
+func getPodCIDRsOnNode(node *corev1.Node) []string {
+	if node.Spec.PodCIDRs != nil {
+		return node.Spec.PodCIDRs
+	}
+
+	if node.Spec.PodCIDR == "" {
+		klog.Errorf("PodCIDR is empty for Node %s", node.Name)
+		// Does not help to return an error and trigger controller retries.
+		return nil
+	}
+	return []string{node.Spec.PodCIDR}
+}
+
 // createIPSecTunnelPort creates an IPSec tunnel port for the remote Node if the
 // tunnel does not exist, and returns the ofport number.
 func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int32, error) {
@@ -545,6 +564,9 @@ func ParseTunnelInterfaceConfig(
 
 // GetNodeAddr gets the available IP address of a Node. GetNodeAddr will first try to get the
 // NodeInternalIP, then try to get the NodeExternalIP.
+// Note: Although K8s supports dual-stack, there is only a single Internal address per Node because of issue (
+// kubernetes/kubernetes#91940 ). The Node might have multiple addresses after the issue is fixed, and one per address
+// family. And we should change the return type at that time.
 func GetNodeAddr(node *corev1.Node) (net.IP, error) {
 	addresses := make(map[corev1.NodeAddressType]string)
 	for _, addr := range node.Status.Addresses {

pkg/agent/flowexporter/connections/conntrack_linux_test.go

@@ -71,7 +71,7 @@ func TestConnTrackSystem_DumpFlows(t *testing.T) {
 	}
 	nodeConfig := &config.NodeConfig{
 		GatewayConfig: gwConfig,
-		PodCIDR: &net.IPNet{
+		PodIPv4CIDR: &net.IPNet{
 			IP:   net.IP{1, 2, 3, 0},
 			Mask: net.IPMask{255, 255, 255, 0},
 		},

pkg/agent/openflow/client.go

@@ -292,8 +292,12 @@ func (c *client) InstallNodeFlows(hostname string,
 	c.replayMutex.RLock()
 	defer c.replayMutex.RUnlock()
 
-	flows := []binding.Flow{
-		c.arpResponderFlow(peerGatewayIP, cookie.Node),
+	var flows []binding.Flow
+
+	if peerGatewayIP.To4() != nil {
+		// Since broadcast is not supported in IPv6, ARP should happen only with IPv4 address, and ARP responder flows
+		// only work for IPv4 addresses.
+		flows = append(flows, c.arpResponderFlow(peerGatewayIP, cookie.Node))
 	}
 	if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPAddr) {
 		flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, peerPodCIDR, tunnelPeerIP, tunOFPort, cookie.Node))

pkg/agent/openflow/client_test.go

@@ -41,9 +41,9 @@ var bridgeMgmtAddr = ofconfig.GetMgmtAddress(ovsconfig.DefaultOVSRunDir, bridgeN
 func installNodeFlows(ofClient Client, cacheKey string) (int, error) {
 	hostName := cacheKey
 	gwMAC, _ := net.ParseMAC("AA:BB:CC:DD:EE:FF")
-	gwIP, IPNet, _ := net.ParseCIDR("10.0.1.1/24")
+	gwIP, ipNet, _ := net.ParseCIDR("10.0.1.1/24")
 	peerNodeIP := net.ParseIP("192.168.1.1")
-	err := ofClient.InstallNodeFlows(hostName, gwMAC, *IPNet, gwIP, peerNodeIP, config.DefaultTunOFPort, 0)
+	err := ofClient.InstallNodeFlows(hostName, gwMAC, *ipNet, gwIP, peerNodeIP, config.DefaultTunOFPort, 0)
 	client := ofClient.(*client)
 	fCacheI, ok := client.nodeFlowCache.Load(hostName)
 	if ok {

pkg/agent/openflow/pipeline.go

@@ -788,7 +788,8 @@ func (c *client) l3FwdFlowToRemote(
 	tunnelPeer net.IP,
 	tunOFPort uint32,
 	category cookie.Category) binding.Flow {
-	return c.pipeline[l3ForwardingTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+	ipProto := parseIPProtocol(peerSubnet.IP)
+	return c.pipeline[l3ForwardingTable].BuildFlow(priorityNormal).MatchProtocol(ipProto).
 		MatchDstIPNet(peerSubnet).
 		Action().DecTTL().
 		// Rewrite src MAC to local gateway MAC and rewrite dst MAC to virtual MAC.
@@ -812,8 +813,9 @@ func (c *client) l3FwdFlowToRemoteViaGW(
 	localGatewayMAC net.HardwareAddr,
 	peerSubnet net.IPNet,
 	category cookie.Category) binding.Flow {
+	ipProto := parseIPProtocol(peerSubnet.IP)
 	l3FwdTable := c.pipeline[l3ForwardingTable]
-	return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+	return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProto).
 		MatchDstIPNet(peerSubnet).
 		Action().DecTTL().
 		Action().SetDstMAC(localGatewayMAC).