[IPv6] Change openflow pipeline for L2 Pod networking

1. Add a new table named IPv6 to handle IPv6 ND Solicitation,
ND advertisement and IPv6 Multicast traffic.

2. Add flows in openflow tables (spoofGuardTable, IPv6,
conntrackTable, conntrackStateTable, conntrackCommitTable,
L2ForwardingOutTable) for handling IPv6 L2 Pod networking.

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/containernetworking/cni v0.7.1
 	github.com/containernetworking/plugins v0.8.2-0.20190724153215-ded2f1757770
-	github.com/contiv/libOpenflow v0.0.0-20200424005919-3a6722c98962
+	github.com/contiv/libOpenflow v0.0.0-20200728044739-7c6534390721
 	github.com/contiv/ofnet v0.0.0-00010101000000-000000000000
 	github.com/coreos/go-iptables v0.4.5
 	github.com/davecgh/go-spew v1.1.1
@@ -62,7 +62,7 @@ require (
 replace (
 	// antrea/plugins/octant/go.mod also has this replacement since replace statement in dependencies
 	// were ignored. We need to change antrea/plugins/octant/go.mod if there is any change here.
-	github.com/contiv/ofnet => github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e
+	github.com/contiv/ofnet => github.com/wenyingd/ofnet v0.0.0-20200728094531-d5b4d75f2cc3
 	// fake.NewSimpleClientset is quite slow when it's initialized with massive objects due to
 	// https://github.com/kubernetes/kubernetes/issues/89574. It takes more than tens of minutes to
 	// init a fake client with 200k objects, which makes it hard to run the NetworkPolicy scale test.

go.sum

@@ -74,8 +74,8 @@ github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK
 github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.8.2-0.20190724153215-ded2f1757770 h1:AuFzUXPFhLlTfxhHMGdCIr7wZOQ0J0dEZLoffU0KfRM=
 github.com/containernetworking/plugins v0.8.2-0.20190724153215-ded2f1757770/go.mod h1:AlmXjbiLJBqvZ4vxkWAqjx1CKHVEoQf0/Ugrvc6Cv70=
-github.com/contiv/libOpenflow v0.0.0-20200424005919-3a6722c98962 h1:J5j3JV+Z9/Hg0ufQwk37uk8ZCJ+YUxEFxekBpIMI/JE=
-github.com/contiv/libOpenflow v0.0.0-20200424005919-3a6722c98962/go.mod h1:DtsPlJOByJZ+MO9YITEGUlbJ/jfh/ef0qeNyBYaeNR4=
+github.com/contiv/libOpenflow v0.0.0-20200728044739-7c6534390721 h1:kYk8terl1B/Nw1YsQwL/EQ6DKa0tpVeh2M8gSmbSAHI=
+github.com/contiv/libOpenflow v0.0.0-20200728044739-7c6534390721/go.mod h1:DtsPlJOByJZ+MO9YITEGUlbJ/jfh/ef0qeNyBYaeNR4=
 github.com/contiv/libovsdb v0.0.0-20170227191248-d0061a53e358 h1:AiA9SKyNXulsU7aAnyka3UFHYOIH00A9HvdIRnDXlg0=
 github.com/contiv/libovsdb v0.0.0-20170227191248-d0061a53e358/go.mod h1:+qKEHaNVPj+wrn5st7TEFH9wcUWCJq5ZBvVKPQwzAeg=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -383,8 +383,8 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e h1:NM4NTe6Z+mF5IYlYAiEdRlY8XcMY4P6VlYqgsBhpojQ=
-github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e/go.mod h1:+g6SfqhTVqeGEmUJ0l4WtCgsL4dflTUJE4k+TPCKqXo=
+github.com/wenyingd/ofnet v0.0.0-20200728094531-d5b4d75f2cc3 h1:Fe47GJc0+uCgKDCxTxXlNw+dTDhCOx390tZZaERYhlg=
+github.com/wenyingd/ofnet v0.0.0-20200728094531-d5b4d75f2cc3/go.mod h1:oF9872TvzJqLzLKDGVMItRLWJHlnwXluuIuNbOP5WKM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

pkg/agent/config/node_config.go

@@ -93,7 +93,7 @@ type NodeConfig struct {
 }
 
 func (n *NodeConfig) String() string {
-	return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s",
+	return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s",
 		n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.GatewayConfig)
 }
 

pkg/agent/openflow/client.go

@@ -473,10 +473,11 @@ func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayMA
 func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
 	flows := []binding.Flow{
 		c.gatewayClassifierFlow(gatewayOFPort, cookie.Default),
-		c.gatewayIPSpoofGuardFlow(gatewayOFPort, cookie.Default),
 		c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default),
 		c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default),
 	}
+	hasIPv6Addr := util.ContainIPv6Addr(gatewayAddrs)
+	flows = append(flows, c.gatewayIPSpoofGuardFlows(gatewayOFPort, hasIPv6Addr, cookie.Default)...)
 
 	// Add ARP SpoofGuard flow for local gateway interface.
 	gwIPv4 := util.GetIPv4Addr(gatewayAddrs)
@@ -524,7 +525,10 @@ func (c *client) initialize() error {
 	if err := c.ofEntryOperations.Add(c.arpNormalFlow(cookie.Default)); err != nil {
 		return fmt.Errorf("failed to install arp normal flow: %v", err)
 	}
-	if err := c.ofEntryOperations.Add(c.l2ForwardOutputFlow(cookie.Default)); err != nil {
+	if err := c.ofEntryOperations.AddAll(c.ipv6Flows(cookie.Default)); err != nil {
+		return fmt.Errorf("failed to install ipv6 flows: %v", err)
+	}
+	if err := c.ofEntryOperations.AddAll(c.l2ForwardOutputFlows(cookie.Default)); err != nil {
 		return fmt.Errorf("failed to install L2 forward output flows: %v", err)
 	}
 	if err := c.ofEntryOperations.AddAll(c.connectionTrackFlows(cookie.Default)); err != nil {

pkg/agent/openflow/pipeline.go

@@ -40,6 +40,7 @@ const (
 	uplinkTable           binding.TableIDType = 5
 	spoofGuardTable       binding.TableIDType = 10
 	arpResponderTable     binding.TableIDType = 20
+	ipv6Table             binding.TableIDType = 21
 	serviceHairpinTable   binding.TableIDType = 29
 	conntrackTable        binding.TableIDType = 30
 	conntrackStateTable   binding.TableIDType = 31
@@ -75,6 +76,11 @@ const (
 	markTrafficFromGateway = 1
 	markTrafficFromLocal   = 2
 	markTrafficFromUplink  = 4
+
+	// IPv6 multicast prefix
+	ipv6MulticastAddr = "FF00::/8"
+	// IPv6 link-local prefix
+	ipv6LinkLocalAddr = "FE80::/10"
 )
 
 var (
@@ -86,6 +92,7 @@ var (
 		{uplinkTable, "Uplink"},
 		{spoofGuardTable, "SpoofGuard"},
 		{arpResponderTable, "ARPResponder"},
+		{ipv6Table, "IPv6"},
 		{serviceHairpinTable, "ServiceHairpin"},
 		{conntrackTable, "ConntrackZone"},
 		{conntrackStateTable, "ConntrackState"},
@@ -164,7 +171,8 @@ const (
 	// the selection result needs to be cached.
 	marksRegServiceNeedLearn uint32 = 0b011
 
-	CtZone = 0xfff0
+	CtZone   = 0xfff0
+	CtZoneV6 = 0xffe6
 
 	portFoundMark    = 0b1
 	snatRequiredMark = 0b1
@@ -478,6 +486,10 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow {
 				Action().CT(false, connectionTrackTable.GetNext(), CtZone).CTDone().
 				Cookie(c.cookieAllocator.Request(category).Raw()).
 				Done(),
+			connectionTrackTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
+				Action().CT(false, connectionTrackTable.GetNext(), CtZoneV6).CTDone().
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done(),
 		)
 	}
 	return append(flows,
@@ -493,6 +505,11 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow {
 			Action().Drop().
 			Cookie(c.cookieAllocator.Request(category).Raw()).
 			Done(),
+		connectionTrackStateTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6).
+			MatchCTStateInv(true).MatchCTStateTrk(true).
+			Action().Drop().
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done(),
 		connectionTrackCommitTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
 			MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}).
 			MatchCTStateNew(true).MatchCTStateTrk(true).
@@ -504,6 +521,11 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow {
 			Action().CT(true, connectionTrackCommitTable.GetNext(), CtZone).CTDone().
 			Cookie(c.cookieAllocator.Request(category).Raw()).
 			Done(),
+		connectionTrackCommitTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6).
+			MatchCTStateNew(true).MatchCTStateTrk(true).
+			Action().CT(true, connectionTrackCommitTable.GetNext(), CtZoneV6).CTDone().
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done(),
 	)
 }
 
@@ -558,15 +580,6 @@ func (c *client) l2ForwardCalcFlow(dstMAC net.HardwareAddr, ofPort uint32, categ
 		Done()
 }
 
-// l2ForwardOutputFlow generates the flow that outputs packets to OVS port after L2 forwarding calculation.
-func (c *client) l2ForwardOutputFlow(category cookie.Category) binding.Flow {
-	return c.pipeline[L2ForwardingOutTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
-		MatchRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
-		Action().OutputRegRange(int(portCacheReg), ofPortRegRange).
-		Cookie(c.cookieAllocator.Request(category).Raw()).
-		Done()
-}
-
 // traceflowL2ForwardOutputFlow generates Traceflow specific flow that outputs traceflow packets to OVS port and Antrea
 // Agent after L2forwarding calculation.
 func (c *client) traceflowL2ForwardOutputFlow(dataplaneTag uint8, category cookie.Category) binding.Flow {
@@ -594,6 +607,24 @@ func (c *client) l2ForwardOutputServiceHairpinFlow() binding.Flow {
 		Done()
 }
 
+// l2ForwardOutputFlows generates the flow that outputs packets to OVS port after L2 forwarding calculation.
+func (c *client) l2ForwardOutputFlows(category cookie.Category) []binding.Flow {
+	var flows []binding.Flow
+	flows = append(flows,
+		c.pipeline[L2ForwardingOutTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+			MatchRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
+			Action().OutputRegRange(int(portCacheReg), ofPortRegRange).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done(),
+		c.pipeline[L2ForwardingOutTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
+			MatchRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
+			Action().OutputRegRange(int(portCacheReg), ofPortRegRange).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done(),
+	)
+	return flows
+}
+
 // l3FlowsToPod generates the flow to rewrite MAC if the packet is received from tunnel port and destined for local Pods.
 func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIPs []net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) []binding.Flow {
 	l3FwdTable := c.pipeline[l3ForwardingTable]
@@ -751,14 +782,23 @@ func (c *client) podIPSpoofGuardFlow(ifIPs []net.IP, ifMAC net.HardwareAddr, ifO
 	var flows []binding.Flow
 	for _, ifIP := range ifIPs {
 		ipProtocol := parseIPProtocol(ifIP)
-		flow := ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
-			MatchInPort(ifOFPort).
-			MatchSrcMAC(ifMAC).
-			MatchSrcIP(ifIP).
-			Action().GotoTable(ipSpoofGuardTable.GetNext()).
-			Cookie(c.cookieAllocator.Request(category).Raw()).
-			Done()
-		flows = append(flows, flow)
+		if ipProtocol == binding.ProtocolIP {
+			flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
+				MatchInPort(ifOFPort).
+				MatchSrcMAC(ifMAC).
+				MatchSrcIP(ifIP).
+				Action().GotoTable(ipSpoofGuardTable.GetNext()).
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done())
+		} else if ipProtocol == binding.ProtocolIPv6 {
+			flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
+				MatchInPort(ifOFPort).
+				MatchSrcMAC(ifMAC).
+				MatchSrcIP(ifIP).
+				Action().GotoTable(ipv6Table).
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done())
+		}
 	}
 	return flows
 }
@@ -807,17 +847,6 @@ func (c *client) arpSpoofGuardFlow(ifIP net.IP, ifMAC net.HardwareAddr, ifOFPort
 		Done()
 }
 
-// gatewayIPSpoofGuardFlow generates the flow to skip spoof guard checking for traffic sent from gateway interface.
-func (c *client) gatewayIPSpoofGuardFlow(gatewayOFPort uint32, category cookie.Category) binding.Flow {
-	ipPipeline := c.pipeline
-	ipSpoofGuardTable := ipPipeline[spoofGuardTable]
-	return ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
-		MatchInPort(gatewayOFPort).
-		Action().GotoTable(ipSpoofGuardTable.GetNext()).
-		Cookie(c.cookieAllocator.Request(category).Raw()).
-		Done()
-}
-
 // sessionAffinityReselectFlow generates the flow which resubmits the service accessing
 // packet back to serviceLBTable if there is no endpointDNAT flow matched. This
 // case will occur if an Endpoint is removed and is the learned Endpoint
@@ -831,6 +860,26 @@ func (c *client) sessionAffinityReselectFlow() binding.Flow {
 		Done()
 }
 
+// gatewayIPSpoofGuardFlow generates the flow to skip spoof guard checking for traffic sent from gateway interface.
+func (c *client) gatewayIPSpoofGuardFlows(gatewayOFPort uint32, hasIPv6Addr bool, category cookie.Category) []binding.Flow {
+	ipPipeline := c.pipeline
+	ipSpoofGuardTable := ipPipeline[spoofGuardTable]
+	var flows []binding.Flow
+	flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+		MatchInPort(gatewayOFPort).
+		Action().GotoTable(ipSpoofGuardTable.GetNext()).
+		Cookie(c.cookieAllocator.Request(category).Raw()).
+		Done())
+	if hasIPv6Addr {
+		flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
+			MatchInPort(gatewayOFPort).
+			Action().GotoTable(ipv6Table).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done())
+	}
+	return flows
+}
+
 // serviceCIDRDNATFlow generates flows to match dst IP in service CIDR and output to host gateway interface directly.
 func (c *client) serviceCIDRDNATFlow(serviceCIDR *net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) binding.Flow {
 	return c.pipeline[dnatTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
@@ -859,6 +908,47 @@ func (c *client) arpNormalFlow(category cookie.Category) binding.Flow {
 		Done()
 }
 
+// ipv6Flows generates the flows to allow IPv6 packets from link-local addresses and
+// handle multicast packets, Neighbor Solicitation and ND Advertisement packets properly.
+func (c *client) ipv6Flows(category cookie.Category) []binding.Flow {
+	var flows []binding.Flow
+	// TODO: Remove the flag after finishing Antrea Proxy changes
+	if !c.enableProxy {
+		_, ipv6LinkLocalIpnet, _ := net.ParseCIDR(ipv6LinkLocalAddr)
+		_, ipv6MulticastIpnet, _ := net.ParseCIDR(ipv6MulticastAddr)
+		flows = append(flows,
+			// Allow IPv6 packets (e.g. Multicast Listener Report Message V2) which are sent from link-local addresses in spoofGuardTable,
+			// so that these packets will not be dropped.
+			c.pipeline[spoofGuardTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
+				MatchSrcIPNet(*ipv6LinkLocalIpnet).
+				Action().GotoTable(ipv6Table).
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done(),
+			// Handle IPv6 Neighbor Solicitation and Neighbor Advertisement as a regular L2 learning Switch by using normal.
+			c.pipeline[ipv6Table].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolICMPv6).
+				MatchICMPv6Type(135).
+				MatchICMPv6Code(0).
+				Action().Normal().
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done(),
+			c.pipeline[ipv6Table].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolICMPv6).
+				MatchICMPv6Type(136).
+				MatchICMPv6Code(0).
+				Action().Normal().
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done(),
+			// Handle IPv6 multicast packets as a regular L2 learning Switch by using normal.
+			// It is used to ensure that all kinds of IPv6 multicast packets are properly handled (e.g. Multicast Listener Report Message V2).
+			c.pipeline[ipv6Table].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
+				MatchDstIPNet(*ipv6MulticastIpnet).
+				Action().Normal().
+				Cookie(c.cookieAllocator.Request(category).Raw()).
+				Done(),
+		)
+	}
+	return flows
+}
+
 // conjunctionActionFlow generates the flow to jump to a specific table if policyRuleConjunction ID is matched. Priority of
 // conjunctionActionFlow is created at priorityLow for k8s network policies, and *priority assigned by PriorityAssigner for CNP.
 func (c *client) conjunctionActionFlow(conjunctionID uint32, tableID binding.TableIDType, nextTable binding.TableIDType, priority *uint16) binding.Flow {
@@ -1333,6 +1423,7 @@ func generatePipeline(bridge binding.Bridge, enableProxy bool) map[binding.Table
 		ClassifierTable:       bridge.CreateTable(ClassifierTable, spoofGuardTable, binding.TableMissActionDrop),
 		spoofGuardTable:       bridge.CreateTable(spoofGuardTable, conntrackTable, binding.TableMissActionDrop),
 		arpResponderTable:     bridge.CreateTable(arpResponderTable, binding.LastTableID, binding.TableMissActionDrop),
+		ipv6Table:             bridge.CreateTable(ipv6Table, conntrackTable, binding.TableMissActionNext),
 		conntrackTable:        bridge.CreateTable(conntrackTable, conntrackStateTable, binding.TableMissActionNone),
 		conntrackStateTable:   bridge.CreateTable(conntrackStateTable, dnatTable, binding.TableMissActionNext),
 		dnatTable:             bridge.CreateTable(dnatTable, cnpEgressRuleTable, binding.TableMissActionNext),

pkg/agent/util/net.go

@@ -127,3 +127,12 @@ func GetIPv4Addr(ips []net.IP) net.IP {
 	}
 	return nil
 }
+
+func ContainIPv6Addr(ips []net.IP) bool {
+	for _, ip := range ips {
+		if ip.To4() == nil {
+			return true
+		}
+	}
+	return false
+}

pkg/ovs/openflow/interfaces.go

@@ -226,6 +226,8 @@ type FlowBuilder interface {
 	MatchTCPDstPort(port uint16) FlowBuilder
 	MatchUDPDstPort(port uint16) FlowBuilder
 	MatchSCTPDstPort(port uint16) FlowBuilder
+	MatchICMPv6Type(icmp6Type byte) FlowBuilder
+	MatchICMPv6Code(icmp6Code byte) FlowBuilder
 	MatchTunMetadata(index int, data uint32) FlowBuilder
 	// MatchCTSrcIP matches the source IPv4 address of the connection tracker original direction tuple.
 	MatchCTSrcIP(ip net.IP) FlowBuilder