Skip to content

Commit

Permalink
[IPv6] Change openflow pipeline for L2 Pod networking (#1040)
Browse files Browse the repository at this point in the history 
1. Add a new table named IPv6 to handle IPv6 ND Solicitation,
ND advertisement and IPv6 Multicast traffic.

2. Add flows in openflow tables (spoofGuardTable, IPv6,
conntrackTable, conntrackStateTable, conntrackCommitTable,
L2ForwardingOutTable) for handling IPv6 L2 Pod networking.
  • Loading branch information
@mengdie-song @wenyingd
mengdie-song authored and wenyingd committed Sep 25, 2020
1 parent d801238 commit 2193c16
Show file tree
Hide file tree
Showing 9 changed files with 216 additions and 53 deletions.
2 changes: 1 addition & 1 deletion pkg/agent/config/node_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ type NodeConfig struct {
}

func (n *NodeConfig) String() string {
return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s",
return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s",
n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.GatewayConfig)
}

Expand Down
8 changes: 6 additions & 2 deletions pkg/agent/openflow/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,10 +476,11 @@ func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayOF
func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
flows := []binding.Flow{
c.gatewayClassifierFlow(gatewayOFPort, cookie.Default),
c.gatewayIPSpoofGuardFlow(gatewayOFPort, cookie.Default),
c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default),
c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default),
}
hasIPv6Addr := util.ContainIPv6Addr(gatewayAddrs)
flows = append(flows, c.gatewayIPSpoofGuardFlows(gatewayOFPort, hasIPv6Addr, cookie.Default)...)

// Add ARP SpoofGuard flow for local gateway interface.
gwIPv4 := util.GetIPv4Addr(gatewayAddrs)
Expand Down Expand Up @@ -527,7 +528,10 @@ func (c *client) initialize() error {
if err := c.ofEntryOperations.Add(c.arpNormalFlow(cookie.Default)); err != nil {
return fmt.Errorf("failed to install arp normal flow: %v", err)
}
if err := c.ofEntryOperations.Add(c.l2ForwardOutputFlow(cookie.Default)); err != nil {
if err := c.ofEntryOperations.AddAll(c.ipv6Flows(cookie.Default)); err != nil {
return fmt.Errorf("failed to install ipv6 flows: %v", err)
}
if err := c.ofEntryOperations.AddAll(c.l2ForwardOutputFlows(cookie.Default)); err != nil {
return fmt.Errorf("failed to install L2 forward output flows: %v", err)
}
if err := c.ofEntryOperations.AddAll(c.connectionTrackFlows(cookie.Default)); err != nil {
Expand Down
167 changes: 130 additions & 37 deletions pkg/agent/openflow/pipeline.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ const (
uplinkTable binding.TableIDType = 5
spoofGuardTable binding.TableIDType = 10
arpResponderTable binding.TableIDType = 20
ipv6Table binding.TableIDType = 21
serviceHairpinTable binding.TableIDType = 29
conntrackTable binding.TableIDType = 30
conntrackStateTable binding.TableIDType = 31
Expand Down Expand Up @@ -79,6 +80,11 @@ const (
markTrafficFromGateway = 1
markTrafficFromLocal = 2
markTrafficFromUplink = 4

// IPv6 multicast prefix
ipv6MulticastAddr = "FF00::/8"
// IPv6 link-local prefix
ipv6LinkLocalAddr = "FE80::/10"
)

var (
Expand All @@ -99,6 +105,7 @@ var (
{uplinkTable, "Uplink"},
{spoofGuardTable, "SpoofGuard"},
{arpResponderTable, "ARPResponder"},
{ipv6Table, "IPv6"},
{serviceHairpinTable, "ServiceHairpin"},
{conntrackTable, "ConntrackZone"},
{conntrackStateTable, "ConntrackState"},
Expand Down Expand Up @@ -212,7 +219,8 @@ const (
// the selection result needs to be cached.
marksRegServiceNeedLearn uint32 = 0b011

CtZone = 0xfff0
CtZone = 0xfff0
CtZoneV6 = 0xffe6

portFoundMark = 0b1
snatRequiredMark = 0b1
Expand Down Expand Up @@ -533,6 +541,10 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow {
Action().CT(false, connectionTrackTable.GetNext(), CtZone).CTDone().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
connectionTrackTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
Action().CT(false, connectionTrackTable.GetNext(), CtZoneV6).CTDone().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
)
}
return append(flows,
Expand All @@ -548,6 +560,11 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow {
Action().Drop().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
connectionTrackStateTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6).
MatchCTStateInv(true).MatchCTStateTrk(true).
Action().Drop().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
connectionTrackCommitTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}).
MatchCTStateNew(true).MatchCTStateTrk(true).
Expand All @@ -559,6 +576,11 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow {
Action().CT(true, connectionTrackCommitTable.GetNext(), CtZone).CTDone().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
connectionTrackCommitTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6).
MatchCTStateNew(true).MatchCTStateTrk(true).
Action().CT(true, connectionTrackCommitTable.GetNext(), CtZoneV6).CTDone().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
)
}

Expand Down Expand Up @@ -620,15 +642,6 @@ func (c *client) l2ForwardCalcFlow(dstMAC net.HardwareAddr, ofPort uint32, categ
Done()
}

// l2ForwardOutputFlow generates the flow that outputs packets to OVS port after L2 forwarding calculation.
func (c *client) l2ForwardOutputFlow(category cookie.Category) binding.Flow {
return c.pipeline[L2ForwardingOutTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
MatchRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
Action().OutputRegRange(int(portCacheReg), ofPortRegRange).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done()
}

// traceflowL2ForwardOutputFlow generates Traceflow specific flow that outputs traceflow packets to OVS port and Antrea
// Agent after L2forwarding calculation.
func (c *client) traceflowL2ForwardOutputFlow(dataplaneTag uint8, category cookie.Category) binding.Flow {
Expand Down Expand Up @@ -656,6 +669,24 @@ func (c *client) l2ForwardOutputServiceHairpinFlow() binding.Flow {
Done()
}

// l2ForwardOutputFlows generates the flow that outputs packets to OVS port after L2 forwarding calculation.
func (c *client) l2ForwardOutputFlows(category cookie.Category) []binding.Flow {
var flows []binding.Flow
flows = append(flows,
c.pipeline[L2ForwardingOutTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
MatchRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
Action().OutputRegRange(int(portCacheReg), ofPortRegRange).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
c.pipeline[L2ForwardingOutTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
MatchRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
Action().OutputRegRange(int(portCacheReg), ofPortRegRange).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
)
return flows
}

// l3FlowsToPod generates the flow to rewrite MAC if the packet is received from tunnel port and destined for local Pods.
func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIPs []net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) []binding.Flow {
l3FwdTable := c.pipeline[l3ForwardingTable]
Expand Down Expand Up @@ -813,14 +844,23 @@ func (c *client) podIPSpoofGuardFlow(ifIPs []net.IP, ifMAC net.HardwareAddr, ifO
var flows []binding.Flow
for _, ifIP := range ifIPs {
ipProtocol := parseIPProtocol(ifIP)
flow := ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
MatchInPort(ifOFPort).
MatchSrcMAC(ifMAC).
MatchSrcIP(ifIP).
Action().GotoTable(ipSpoofGuardTable.GetNext()).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done()
flows = append(flows, flow)
if ipProtocol == binding.ProtocolIP {
flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
MatchInPort(ifOFPort).
MatchSrcMAC(ifMAC).
MatchSrcIP(ifIP).
Action().GotoTable(ipSpoofGuardTable.GetNext()).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done())
} else if ipProtocol == binding.ProtocolIPv6 {
flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
MatchInPort(ifOFPort).
MatchSrcMAC(ifMAC).
MatchSrcIP(ifIP).
Action().GotoTable(ipv6Table).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done())
}
}
return flows
}
Expand Down Expand Up @@ -869,17 +909,6 @@ func (c *client) arpSpoofGuardFlow(ifIP net.IP, ifMAC net.HardwareAddr, ifOFPort
Done()
}

// gatewayIPSpoofGuardFlow generates the flow to skip spoof guard checking for traffic sent from gateway interface.
func (c *client) gatewayIPSpoofGuardFlow(gatewayOFPort uint32, category cookie.Category) binding.Flow {
ipPipeline := c.pipeline
ipSpoofGuardTable := ipPipeline[spoofGuardTable]
return ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
MatchInPort(gatewayOFPort).
Action().GotoTable(ipSpoofGuardTable.GetNext()).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done()
}

// sessionAffinityReselectFlow generates the flow which resubmits the service accessing
// packet back to serviceLBTable if there is no endpointDNAT flow matched. This
// case will occur if an Endpoint is removed and is the learned Endpoint
Expand All @@ -893,6 +922,26 @@ func (c *client) sessionAffinityReselectFlow() binding.Flow {
Done()
}

// gatewayIPSpoofGuardFlow generates the flow to skip spoof guard checking for traffic sent from gateway interface.
func (c *client) gatewayIPSpoofGuardFlows(gatewayOFPort uint32, hasIPv6Addr bool, category cookie.Category) []binding.Flow {
ipPipeline := c.pipeline
ipSpoofGuardTable := ipPipeline[spoofGuardTable]
var flows []binding.Flow
flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
MatchInPort(gatewayOFPort).
Action().GotoTable(ipSpoofGuardTable.GetNext()).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done())
if hasIPv6Addr {
flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
MatchInPort(gatewayOFPort).
Action().GotoTable(ipv6Table).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done())
}
return flows
}

// serviceCIDRDNATFlow generates flows to match dst IP in service CIDR and output to host gateway interface directly.
func (c *client) serviceCIDRDNATFlow(serviceCIDR *net.IPNet, gatewayOFPort uint32) binding.Flow {
return c.pipeline[dnatTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
Expand Down Expand Up @@ -967,6 +1016,47 @@ func (c *client) dropRuleMetricFlow(conjunctionID uint32, ingress bool) binding.
Done()
}

// ipv6Flows generates the flows to allow IPv6 packets from link-local addresses and
// handle multicast packets, Neighbor Solicitation and ND Advertisement packets properly.
func (c *client) ipv6Flows(category cookie.Category) []binding.Flow {
var flows []binding.Flow
// TODO: Remove the flag after finishing Antrea Proxy changes
if !c.enableProxy {
_, ipv6LinkLocalIpnet, _ := net.ParseCIDR(ipv6LinkLocalAddr)
_, ipv6MulticastIpnet, _ := net.ParseCIDR(ipv6MulticastAddr)
flows = append(flows,
// Allow IPv6 packets (e.g. Multicast Listener Report Message V2) which are sent from link-local addresses in spoofGuardTable,
// so that these packets will not be dropped.
c.pipeline[spoofGuardTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
MatchSrcIPNet(*ipv6LinkLocalIpnet).
Action().GotoTable(ipv6Table).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
// Handle IPv6 Neighbor Solicitation and Neighbor Advertisement as a regular L2 learning Switch by using normal.
c.pipeline[ipv6Table].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolICMPv6).
MatchICMPv6Type(135).
MatchICMPv6Code(0).
Action().Normal().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
c.pipeline[ipv6Table].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolICMPv6).
MatchICMPv6Type(136).
MatchICMPv6Code(0).
Action().Normal().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
// Handle IPv6 multicast packets as a regular L2 learning Switch by using normal.
// It is used to ensure that all kinds of IPv6 multicast packets are properly handled (e.g. Multicast Listener Report Message V2).
c.pipeline[ipv6Table].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6).
MatchDstIPNet(*ipv6MulticastIpnet).
Action().Normal().
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
)
}
return flows
}

// conjunctionActionFlow generates the flow to jump to a specific table if policyRuleConjunction ID is matched. Priority of
// conjunctionActionFlow is created at priorityLow for k8s network policies, and *priority assigned by PriorityAssigner for AntreaPolicy.
func (c *client) conjunctionActionFlow(conjunctionID uint32, tableID binding.TableIDType, nextTable binding.TableIDType, priority *uint16) binding.Flow {
Expand Down Expand Up @@ -1450,6 +1540,7 @@ func generatePipeline(bridge binding.Bridge, enableProxy, enableAntreaNP bool) m
uplinkTable: bridge.CreateTable(uplinkTable, spoofGuardTable, binding.TableMissActionNone),
spoofGuardTable: bridge.CreateTable(spoofGuardTable, serviceHairpinTable, binding.TableMissActionDrop),
arpResponderTable: bridge.CreateTable(arpResponderTable, binding.LastTableID, binding.TableMissActionDrop),
ipv6Table: bridge.CreateTable(ipv6Table, serviceHairpinTable, binding.TableMissActionNext),
serviceHairpinTable: bridge.CreateTable(serviceHairpinTable, conntrackTable, binding.TableMissActionNext),
conntrackTable: bridge.CreateTable(conntrackTable, conntrackStateTable, binding.TableMissActionNone),
conntrackStateTable: bridge.CreateTable(conntrackStateTable, endpointDNATTable, binding.TableMissActionNext),
Expand All @@ -1470,15 +1561,17 @@ func generatePipeline(bridge binding.Bridge, enableProxy, enableAntreaNP bool) m
}
} else {
pipeline = map[binding.TableIDType]binding.Table{
ClassifierTable: bridge.CreateTable(ClassifierTable, spoofGuardTable, binding.TableMissActionDrop),
spoofGuardTable: bridge.CreateTable(spoofGuardTable, conntrackTable, binding.TableMissActionDrop),
arpResponderTable: bridge.CreateTable(arpResponderTable, binding.LastTableID, binding.TableMissActionDrop),
conntrackTable: bridge.CreateTable(conntrackTable, conntrackStateTable, binding.TableMissActionNone),
conntrackStateTable: bridge.CreateTable(conntrackStateTable, dnatTable, binding.TableMissActionNext),
dnatTable: bridge.CreateTable(dnatTable, egressEntryTable, binding.TableMissActionNext),
EgressRuleTable: bridge.CreateTable(EgressRuleTable, EgressDefaultTable, binding.TableMissActionNext),
ClassifierTable: bridge.CreateTable(ClassifierTable, spoofGuardTable, binding.TableMissActionDrop),
spoofGuardTable: bridge.CreateTable(spoofGuardTable, conntrackTable, binding.TableMissActionDrop),
arpResponderTable: bridge.CreateTable(arpResponderTable, binding.LastTableID, binding.TableMissActionDrop),
ipv6Table: bridge.CreateTable(ipv6Table, conntrackTable, binding.TableMissActionNext),
conntrackTable: bridge.CreateTable(conntrackTable, conntrackStateTable, binding.TableMissActionNone),
conntrackStateTable: bridge.CreateTable(conntrackStateTable, dnatTable, binding.TableMissActionNext),
dnatTable: bridge.CreateTable(dnatTable, egressEntryTable, binding.TableMissActionNext),
EgressRuleTable: bridge.CreateTable(EgressRuleTable, EgressDefaultTable, binding.TableMissActionNext),
EgressDefaultTable: bridge.CreateTable(EgressDefaultTable, EgressMetricTable, binding.TableMissActionNext),
EgressMetricTable: bridge.CreateTable(EgressMetricTable, l3ForwardingTable, binding.TableMissActionNext), l3ForwardingTable: bridge.CreateTable(l3ForwardingTable, l2ForwardingCalcTable, binding.TableMissActionNext),
EgressMetricTable: bridge.CreateTable(EgressMetricTable, l3ForwardingTable, binding.TableMissActionNext),
l3ForwardingTable: bridge.CreateTable(l3ForwardingTable, l2ForwardingCalcTable, binding.TableMissActionNext),
l2ForwardingCalcTable: bridge.CreateTable(l2ForwardingCalcTable, IngressEntryTable, binding.TableMissActionNext),
IngressRuleTable: bridge.CreateTable(IngressRuleTable, IngressDefaultTable, binding.TableMissActionNext),
IngressDefaultTable: bridge.CreateTable(IngressDefaultTable, IngressMetricTable, binding.TableMissActionNext),
Expand Down
9 changes: 9 additions & 0 deletions pkg/agent/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,12 @@ func GetIPv4Addr(ips []net.IP) net.IP {
}
return nil
}

func ContainIPv6Addr(ips []net.IP) bool {
for _, ip := range ips {
if ip.To4() == nil {
return true
}
}
return false
}
2 changes: 2 additions & 0 deletions pkg/ovs/openflow/interfaces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,8 @@ type FlowBuilder interface {
MatchTCPDstPort(port uint16) FlowBuilder
MatchUDPDstPort(port uint16) FlowBuilder
MatchSCTPDstPort(port uint16) FlowBuilder
MatchICMPv6Type(icmp6Type byte) FlowBuilder
MatchICMPv6Code(icmp6Code byte) FlowBuilder
MatchTunMetadata(index int, data uint32) FlowBuilder
// MatchCTSrcIP matches the source IPv4 address of the connection tracker original direction tuple.
MatchCTSrcIP(ip net.IP) FlowBuilder
Expand Down
Loading

0 comments on commit 2193c16

Please sign in to comment.