diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 8588bb07e50..44b3c9d3875 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -94,7 +94,8 @@ func run(o *Options) error { ovsBridgeMgmtAddr := ofconfig.GetMgmtAddress(o.config.OVSRunDir, o.config.OVSBridge) ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr, ovsDatapathType, features.DefaultFeatureGate.Enabled(features.AntreaProxy), - features.DefaultFeatureGate.Enabled(features.AntreaPolicy)) + features.DefaultFeatureGate.Enabled(features.AntreaPolicy), + false) _, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR) var serviceCIDRNetv6 *net.IPNet diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index ccd67c32abd..d00042a2638 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -294,9 +294,9 @@ func (i *Initializer) initOpenFlowPipeline() error { return err } - // On Windows platform, extra flows are needed to perform SNAT for the - // traffic to external network. - if err := i.initExternalConnectivityFlows(); err != nil { + // Install OpenFlow entries to enable Pod traffic to external IP + // addresses. + if err := i.ofClient.InstallExternalFlows(); err != nil { klog.Errorf("Failed to install openflow entries for external connectivity: %v", err) return err } diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index d6b899ee3c4..e9158572e92 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -35,11 +35,6 @@ func (i *Initializer) initHostNetworkFlows() error { return nil } -// initExternalConnectivityFlows returns immediately on Linux. The corresponding functions are provided in routeClient. -func (i *Initializer) initExternalConnectivityFlows() error { - return nil -} - // getTunnelLocalIP returns local_ip of tunnel port. // On linux platform, local_ip option is not needed. func (i *Initializer) getTunnelPortLocalIP() net.IP { diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 799220d02ba..13a8ab745c5 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -188,19 +188,6 @@ func (i *Initializer) initHostNetworkFlows() error { return nil } -// initExternalConnectivityFlows installs OpenFlow entries to SNAT Pod traffic -// using Node IP, and then Pod could communicate to the external IP addresses. -func (i *Initializer) initExternalConnectivityFlows() error { - if i.nodeConfig.PodIPv4CIDR == nil { - return fmt.Errorf("Failed to find valid IPv4 PodCIDR") - } - // Install OpenFlow entries on the OVS to enable Pod traffic to communicate to external IP addresses. - if err := i.ofClient.InstallExternalFlows(); err != nil { - return err - } - return nil -} - // getTunnelLocalIP returns local_ip of tunnel port func (i *Initializer) getTunnelPortLocalIP() net.IP { return i.nodeConfig.NodeIPAddr.IP diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index a9080339804..ff121893dc8 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -150,10 +150,11 @@ type Client interface { // This function is only used for Windows platform. InstallBridgeUplinkFlows() error - // InstallExternalFlows sets up flows to enable Pods to communicate to the external IP addresses. The corresponding - // OpenFlow entries include: 1) identify the packets from local Pods to the external IP address, 2) mark the traffic - // in the connection tracking context, and 3) SNAT the packets with Node IP. - // This function is only used for Windows platform. + // InstallExternalFlows sets up flows to enable Pods to communicate to + // the external IP addresses. The flows identify the packets from local + // Pods to the external IP address, and mark the packets to be SNAT'd + // with the configured SNAT IPs. On Windows Node, the flows also perform + // SNAT with the Openflow NAT action. InstallExternalFlows() error // Disconnect disconnects the connection between client and OFSwitch. @@ -476,22 +477,6 @@ func (c *client) UninstallServiceFlows(svcIP net.IP, svcPort uint16, protocol bi return c.deleteFlows(c.serviceFlowCache, cacheKey) } -func (c *client) InstallLoadBalancerServiceFromOutsideFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error { - c.replayMutex.RLock() - defer c.replayMutex.RUnlock() - var flows []binding.Flow - flows = append(flows, c.loadBalancerServiceFromOutsideFlow(svcIP, svcPort, protocol)) - cacheKey := fmt.Sprintf("LoadBalancerService_%s_%d_%s", svcIP, svcPort, protocol) - return c.addFlows(c.serviceFlowCache, cacheKey, flows) -} - -func (c *client) UninstallLoadBalancerServiceFromOutsideFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error { - c.replayMutex.RLock() - defer c.replayMutex.RUnlock() - cacheKey := fmt.Sprintf("LoadBalancerService_%s_%d_%s", svcIP, svcPort, protocol) - return c.deleteFlows(c.serviceFlowCache, cacheKey) -} - func (c *client) GetServiceFlowKeys(svcIP net.IP, svcPort uint16, protocol binding.Protocol, endpoints []proxy.Endpoint) []string { cacheKey := generateServicePortFlowCacheKey(svcIP, svcPort, protocol) flowKeys := c.getFlowKeysFromCache(c.serviceFlowCache, cacheKey) @@ -579,16 +564,6 @@ func (c *client) InstallDefaultTunnelFlows() error { return nil } -func (c *client) InstallBridgeUplinkFlows() error { - flows := c.hostBridgeUplinkFlows(*c.nodeConfig.PodIPv4CIDR, cookie.Default) - c.hostNetworkingFlows = flows - if err := c.ofEntryOperations.AddAll(flows); err != nil { - return err - } - c.hostNetworkingFlows = flows - return nil -} - func (c *client) initialize() error { if err := c.ofEntryOperations.AddAll(c.defaultFlows()); err != nil { return fmt.Errorf("failed to install default flows: %v", err) @@ -655,9 +630,16 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo func (c *client) InstallExternalFlows() error { nodeIP := c.nodeConfig.NodeIPAddr.IP - podSubnet := c.nodeConfig.PodIPv4CIDR - flows := c.uplinkSNATFlows(cookie.SNAT) - flows = append(flows, c.snatFlows(nodeIP, *podSubnet, cookie.SNAT)...) + localGatewayMAC := c.nodeConfig.GatewayConfig.MAC + + var flows []binding.Flow + if c.nodeConfig.PodIPv4CIDR != nil { + flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + } + if c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + } + if err := c.ofEntryOperations.AddAll(flows); err != nil { return fmt.Errorf("failed to install flows for external communication: %v", err) } diff --git a/pkg/agent/openflow/client_other.go b/pkg/agent/openflow/client_other.go new file mode 100644 index 00000000000..acf97b622cd --- /dev/null +++ b/pkg/agent/openflow/client_other.go @@ -0,0 +1,36 @@ +// +build !windows +// package openflow is needed by antctl which is compiled for macOS too. + +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "net" + + binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" +) + +func (c *client) InstallBridgeUplinkFlows() error { + return nil +} + +func (c *client) InstallLoadBalancerServiceFromOutsideFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error { + return nil +} + +func (c *client) UninstallLoadBalancerServiceFromOutsideFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error { + return nil +} diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 40f94db242a..1dfad0f4d63 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -101,7 +101,7 @@ func TestIdempotentFlowInstallation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -129,7 +129,7 @@ func TestIdempotentFlowInstallation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -170,7 +170,7 @@ func TestFlowInstallationFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -204,7 +204,7 @@ func TestConcurrentFlowInstallation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -467,7 +467,7 @@ func Test_client_SendTraceflowPacket(t *testing.T) { } func prepareTraceflowFlow(ctrl *gomock.Controller) *client { - ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, true) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, true, false) c := ofClient.(*client) c.cookieAllocator = cookie.NewAllocator(0) c.nodeConfig = nodeConfig @@ -485,7 +485,7 @@ func prepareTraceflowFlow(ctrl *gomock.Controller) *client { } func prepareSendTraceflowPacket(ctrl *gomock.Controller, success bool) *client { - ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, true) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, ovsconfig.OVSDatapathSystem, true, true, false) c := ofClient.(*client) c.nodeConfig = nodeConfig m := ovsoftest.NewMockBridge(ctrl) diff --git a/pkg/agent/openflow/client_windows.go b/pkg/agent/openflow/client_windows.go new file mode 100644 index 00000000000..abad71d814f --- /dev/null +++ b/pkg/agent/openflow/client_windows.go @@ -0,0 +1,51 @@ +// +build windows +// package openflow is needed by antctl which is compiled for macOS too. + +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "fmt" + "net" + + "github.com/vmware-tanzu/antrea/pkg/agent/openflow/cookie" + binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" +) + +func (c *client) InstallBridgeUplinkFlows() error { + flows := c.hostBridgeUplinkFlows(*c.nodeConfig.PodIPv4CIDR, cookie.Default) + if err := c.ofEntryOperations.AddAll(flows); err != nil { + return err + } + c.hostNetworkingFlows = flows + return nil +} + +func (c *client) InstallLoadBalancerServiceFromOutsideFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error { + c.replayMutex.RLock() + defer c.replayMutex.RUnlock() + var flows []binding.Flow + flows = append(flows, c.loadBalancerServiceFromOutsideFlow(svcIP, svcPort, protocol)) + cacheKey := fmt.Sprintf("L%s%s%x", svcIP, protocol, svcPort) + return c.addFlows(c.serviceFlowCache, cacheKey, flows) +} + +func (c *client) UninstallLoadBalancerServiceFromOutsideFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error { + c.replayMutex.RLock() + defer c.replayMutex.RUnlock() + cacheKey := fmt.Sprintf("L%s%s%x", svcIP, protocol, svcPort) + return c.deleteFlows(c.serviceFlowCache, cacheKey) +} diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 429fe95ad82..3f4dc223333 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -57,7 +57,8 @@ const ( EgressDefaultTable binding.TableIDType = 60 EgressMetricTable binding.TableIDType = 61 l3ForwardingTable binding.TableIDType = 70 - l3DecTTLTable binding.TableIDType = 71 + snatTable binding.TableIDType = 71 + l3DecTTLTable binding.TableIDType = 72 l2ForwardingCalcTable binding.TableIDType = 80 AntreaPolicyIngressRuleTable binding.TableIDType = 85 DefaultTierIngressRuleTable binding.TableIDType = 89 @@ -83,7 +84,6 @@ const ( markTrafficFromGateway = 1 markTrafficFromLocal = 2 markTrafficFromUplink = 4 - markTrafficFromBridge = 5 // IPv6 multicast prefix ipv6MulticastAddr = "FF00::/8" @@ -120,7 +120,9 @@ var ( {EgressRuleTable, "EgressRule"}, {EgressDefaultTable, "EgressDefaultRule"}, {EgressMetricTable, "EgressMetric"}, - {l3ForwardingTable, "l3Forwarding"}, + {l3ForwardingTable, "L3Forwarding"}, + {snatTable, "SNAT"}, + {l3DecTTLTable, "IPTTLDec"}, {l2ForwardingCalcTable, "L2Forwarding"}, {AntreaPolicyIngressRuleTable, "AntreaPolicyIngressRule"}, {IngressRuleTable, "IngressRule"}, @@ -223,23 +225,9 @@ const ( CtZone = 0xfff0 CtZoneV6 = 0xffe6 - // CtZoneSNAT is only used on Windows and only when AntreaProxy is enabled. - // When a Pod access a ClusterIP Service, and the IP of the selected endpoint - // is not in "cluster-cidr". The request packets need to be SNAT'd(set src IP to local Node IP) - // after have been DNAT'd(set dst IP to endpoint IP). - // For example, the endpoint Pod may run in hostNetwork mode and the IP of the endpoint - // will is the current Node IP. - // We need to use a different ct_zone to track the SNAT'd connection because OVS - // does not support doing both DNAT and SNAT in the same ct_zone. - // - // An example of the connection is a Pod accesses kubernetes API service: - // Pod --> DNAT(CtZone) --> SNAT(CtZoneSNAT) --> Endpoint(API server NodeIP) - // Pod <-- unDNAT(CtZone) <-- unSNAT(CtZoneSNAT) <-- Endpoint(API server NodeIP) - CtZoneSNAT = 0xffdc - - portFoundMark = 0b1 - snatRequiredMark = 0b1 - hairpinMark = 0b1 + + portFoundMark = 0b1 + hairpinMark = 0b1 // macRewriteMark indicates the destination and source MACs of the // packet should be rewritten in the l3ForwardingTable. macRewriteMark = 0b1 @@ -248,7 +236,6 @@ const ( // gatewayCTMark is used to to mark connections initiated through the host gateway interface // (i.e. for which the first packet of the connection was received through the gateway). gatewayCTMark = 0x20 - snatCTMark = 0x40 ServiceCTMark = 0x21 // disposition is loaded in marksReg [21] @@ -271,9 +258,6 @@ var ( ofPortMarkRange = binding.Range{16, 16} // ofPortRegRange takes a 32-bit range of register PortCacheReg to cache the ofPort number of the interface. ofPortRegRange = binding.Range{0, 31} - // snatMarkRange takes the 17th bit of register marksReg to indicate if the packet needs to be SNATed with Node's IP - // or not. Its value is 0x1 if yes. - snatMarkRange = binding.Range{17, 17} // hairpinMarkRange takes the 18th bit of register marksReg to indicate // if the packet needs DNAT to virtual IP or not. Its value is 0x1 if yes. hairpinMarkRange = binding.Range{18, 18} @@ -335,6 +319,7 @@ func portToUint16(port int) uint16 { type client struct { enableProxy bool enableAntreaPolicy bool + enableEgress bool roundInfo types.RoundInfo cookieAllocator cookie.Allocator bridge binding.Bridge @@ -419,6 +404,9 @@ func (c *client) Delete(flow binding.Flow) error { } func (c *client) AddAll(flows []binding.Flow) error { + if len(flows) == 0 { + return nil + } startTime := time.Now() defer func() { d := time.Since(startTime) @@ -1579,128 +1567,18 @@ func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Catego return flows } -// hostBridgeUplinkFlows generates the flows that forward traffic between the -// bridge local port and the uplink port to support the host traffic with -// outside. These flows are needed only on Windows Nodes. -func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Category) (flows []binding.Flow) { - bridgeOFPort := uint32(config.BridgeOFPort) - flows = []binding.Flow{ - c.pipeline[ClassifierTable].BuildFlow(priorityNormal). - MatchInPort(config.UplinkOFPort). - Action().LoadRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). - Action().GotoTable(uplinkTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - c.pipeline[ClassifierTable].BuildFlow(priorityNormal). - MatchInPort(config.BridgeOFPort). - Action().LoadRegRange(int(marksReg), markTrafficFromBridge, binding.Range{0, 15}). - Action().GotoTable(uplinkTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - // Output non-IP packets to the bridge port directly. IP packets - // are redirected to conntrackTable in uplinkSNATFlows() (in - // case they need unSNAT). - c.pipeline[uplinkTable].BuildFlow(priorityLow). - MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). - Action().Output(int(bridgeOFPort)). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - // Forward the packet to conntrackTable if it enters the OVS - // pipeline from the bridge interface and is sent to the local - // Pod subnet. Mark the packet to indicate its destination MAC - // should be rewritten to the real MAC in the L3Frowarding - // table. This is for the case a Pod accesses a NodePort Service - // using the local Node's IP, and then the return traffic after - // the kube-proxy processing will enter the bridge from the - // bridge interface (but not the gateway interface. This is - // probably because we do not enable IP forwarding on the bridge - // interface). - c.pipeline[uplinkTable].BuildFlow(priorityHigh). - MatchProtocol(binding.ProtocolIP). - MatchRegRange(int(marksReg), markTrafficFromBridge, binding.Range{0, 15}). - MatchDstIPNet(localSubnet). - Action().LoadRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange). - Action().GotoTable(conntrackTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - // Output other packets from the bridge port to the uplink port - // directly. - c.pipeline[uplinkTable].BuildFlow(priorityLow). - MatchRegRange(int(marksReg), markTrafficFromBridge, binding.Range{0, 15}). - Action().Output(config.UplinkOFPort). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - } - return flows -} - -// uplinkSNATFlows installs flows for traffic from the uplink port that help -// the SNAT implementation of the external traffic. It is for the Windows Nodes -// only. -func (c *client) uplinkSNATFlows(category cookie.Category) []binding.Flow { - ctStateNext := dnatTable - if c.enableProxy { - ctStateNext = endpointDNATTable - } - bridgeOFPort := uint32(config.BridgeOFPort) - flows := []binding.Flow{ - // Mark the packet to indicate its destination MAC should be rewritten to the real MAC in the L3Forwarding - // table, if the packet is a reply to a Pod from an external address. - c.pipeline[conntrackStateTable].BuildFlow(priorityHigh). - MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(false).MatchCTStateTrk(true). - MatchCTMark(snatCTMark, nil). - MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). - Action().LoadRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange). - Action().GotoTable(ctStateNext). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - // Output the non-SNAT packet to the bridge interface directly if it is received from the uplink interface. - c.pipeline[conntrackStateTable].BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). - MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). - Action().Output(int(bridgeOFPort)). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - } - // Forward the IP packets from the uplink interface to - // conntrackTable. This is for unSNAT the traffic from the local - // Pod subnet to the external network. Non-SNAT packets will be - // output to the bridge port in conntrackStateTable. - if c.enableProxy { - // For the connection which is both applied DNAT and SNAT, the reply packtets - // are received from uplink and need to enter CTZoneSNAT first to do unSNAT. - // Pod --> DNAT(CtZone) --> SNAT(CtZoneSNAT) --> ExternalServer - // Pod <-- unDNAT(CtZone) <-- unSNAT(CtZoneSNAT) <-- ExternalServer - flows = append(flows, c.pipeline[uplinkTable].BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). - MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). - Action().CT(false, conntrackTable, CtZoneSNAT).NAT().CTDone(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) - } else { - flows = append(flows, c.pipeline[uplinkTable].BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). - MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). - Action().GotoTable(conntrackTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) - } - return flows -} - -// snatFlows installs flows to perform SNAT for traffic to the external network. -// It is used on Windows Nodes only. -func (c *client) snatFlows(nodeIP net.IP, localSubnet net.IPNet, category cookie.Category) []binding.Flow { - snatIPRange := &binding.IPRange{StartIP: nodeIP, EndIP: nodeIP} +// snatCommonFlows installs the default flows for performing SNAT for traffic to +// the external network. The flows identify the packets to external, and send +// them to snatTable, where SNAT IPs are looked up for the packets. +func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] nextTable := l3FwdTable.GetNext() + ipProto := getIPProtocol(localSubnet.IP) flows := []binding.Flow{ // First install flows for traffic that should bypass SNAT. - // This flow is for traffic to the local Pod subnet. l3FwdTable.BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). + MatchProtocol(ipProto). MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). MatchDstIPNet(localSubnet). Action().GotoTable(nextTable). @@ -1708,7 +1586,7 @@ func (c *client) snatFlows(nodeIP net.IP, localSubnet net.IPNet, category cookie Done(), // This flow is for the traffic to the local Node IP. l3FwdTable.BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). + MatchProtocol(ipProto). MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). MatchDstIP(nodeIP). Action().GotoTable(nextTable). @@ -1724,76 +1602,39 @@ func (c *client) snatFlows(nodeIP net.IP, localSubnet net.IPNet, category cookie // covered by other flows (the flows matching the local and // remote Pod subnets) anyway. l3FwdTable.BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). + MatchProtocol(ipProto). MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). MatchCTMark(gatewayCTMark, nil). Action().GotoTable(nextTable). Cookie(c.cookieAllocator.Request(category).Raw()). Done(), - // Add the SNAT mark on the packet that is not filtered by other - // flow entries in the L3Forwarding table. + // Send the traffic to external to snatTable. l3FwdTable.BuildFlow(priorityLow). - MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(true).MatchCTStateTrk(true). + MatchProtocol(ipProto). MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). - Action().LoadRegRange(int(marksReg), snatRequiredMark, snatMarkRange). - Action().GotoTable(nextTable). + Action().GotoTable(snatTable). Cookie(c.cookieAllocator.Request(category).Raw()). Done(), - // Force IP packet into the conntrack zone with SNAT. If the connection is SNATed, the reply packet should use - // Pod IP as the destination, and then is forwarded to conntrackStateTable. - c.pipeline[conntrackTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - Action().CT(false, conntrackStateTable, CtZone).NAT().CTDone(). + // For the traffic tunneled from remote Nodes, rewrite the + // destination MAC to the gateway interface MAC. + l3FwdTable.BuildFlow(priorityLow). + MatchProtocol(ipProto). + MatchRegRange(int(marksReg), markTrafficFromTunnel, binding.Range{0, 15}). + Action().SetDstMAC(localGatewayMAC). + Action().GotoTable(snatTable). Cookie(c.cookieAllocator.Request(category).Raw()). Done(), - // Redirect the packet into L2ForwardingOutput table after the packet is SNAT'd. A "SNAT" packet has these - // characteristics: 1) the ct_state is "+new+trk", 2) reg0[17] is set to 1; 3) Node IP is used as the target - // source IP in NAT action, 4) ct_mark is set to 0x40 in the conn_track context. - c.pipeline[conntrackCommitTable].BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(true).MatchCTStateTrk(true).MatchCTStateDNAT(false). - MatchRegRange(int(marksReg), snatRequiredMark, snatMarkRange). - Action().CT(true, L2ForwardingOutTable, CtZone). - SNAT(snatIPRange, nil). - LoadToMark(snatCTMark).CTDone(). + + // Drop the traffic from remote Nodes if no matched SNAT policy. + c.pipeline[snatTable].BuildFlow(priorityLow). + MatchProtocol(ipProto). + MatchCTStateNew(true).MatchCTStateTrk(true). + MatchRegRange(int(marksReg), markTrafficFromTunnel, binding.Range{0, 15}). + Action().Drop(). Cookie(c.cookieAllocator.Request(category).Raw()). Done(), } - // The following flows are for both apply DNAT + SNAT for packets. - // If AntreaProxy is disabled, no DNAT happens in OVS pipeline. - if c.enableProxy { - // If the SNAT is needed after DNAT, mark the snatRequiredMark even the connection is not new. - // Because this kind of packets need to enter CtZoneSNAT to make sure the SNAT can be applied - // before leaving the pipeline. - flows = append(flows, l3FwdTable.BuildFlow(priorityLow). - MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(false).MatchCTStateTrk(true).MatchCTStateDNAT(true). - MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). - Action().LoadRegRange(int(marksReg), snatRequiredMark, snatMarkRange). - Action().GotoTable(nextTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) - // If SNAT is needed after DNAT: - // - For new connection: commit to CtZoneSNAT - // - For existing connection: enter CtZoneSNAT to apply SNAT - flows = append(flows, c.pipeline[conntrackCommitTable].BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(true).MatchCTStateTrk(true).MatchCTStateDNAT(true). - MatchRegRange(int(marksReg), snatRequiredMark, snatMarkRange). - Action().CT(true, L2ForwardingOutTable, CtZoneSNAT). - SNAT(snatIPRange, nil). - LoadToMark(snatCTMark).CTDone(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) - flows = append(flows, c.pipeline[conntrackCommitTable].BuildFlow(priorityNormal). - MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(false).MatchCTStateTrk(true).MatchCTStateDNAT(true). - MatchRegRange(int(marksReg), snatRequiredMark, snatMarkRange). - Action().CT(false, L2ForwardingOutTable, CtZoneSNAT).NAT().CTDone(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) - } return flows } @@ -2059,6 +1900,10 @@ func (c *client) generatePipeline() { c.pipeline[dnatTable] = bridge.CreateTable(dnatTable, c.egressEntryTable, binding.TableMissActionNext) c.pipeline[conntrackCommitTable] = bridge.CreateTable(conntrackCommitTable, L2ForwardingOutTable, binding.TableMissActionNext) } + // The default SNAT is implemented with OVS on Windows. + if c.enableEgress || runtime.IsWindowsPlatform() { + c.pipeline[snatTable] = bridge.CreateTable(snatTable, l2ForwardingCalcTable, binding.TableMissActionNext) + } if runtime.IsWindowsPlatform() { c.pipeline[uplinkTable] = bridge.CreateTable(uplinkTable, spoofGuardTable, binding.TableMissActionNone) } @@ -2069,7 +1914,7 @@ func (c *client) generatePipeline() { } // NewClient is the constructor of the Client interface. -func NewClient(bridgeName, mgmtAddr string, ovsDatapathType ovsconfig.OVSDatapathType, enableProxy, enableAntreaPolicy bool) Client { +func NewClient(bridgeName, mgmtAddr string, ovsDatapathType ovsconfig.OVSDatapathType, enableProxy, enableAntreaPolicy, enableEgress bool) Client { bridge := binding.NewOFBridge(bridgeName, mgmtAddr) policyCache := cache.NewIndexer( policyConjKeyFunc, @@ -2079,6 +1924,7 @@ func NewClient(bridgeName, mgmtAddr string, ovsDatapathType ovsconfig.OVSDatapat bridge: bridge, enableProxy: enableProxy, enableAntreaPolicy: enableAntreaPolicy, + enableEgress: enableEgress, nodeFlowCache: newFlowCategoryCache(), podFlowCache: newFlowCategoryCache(), serviceFlowCache: newFlowCategoryCache(), diff --git a/pkg/agent/openflow/pipeline_other.go b/pkg/agent/openflow/pipeline_other.go new file mode 100644 index 00000000000..35e9ee61951 --- /dev/null +++ b/pkg/agent/openflow/pipeline_other.go @@ -0,0 +1,33 @@ +// +build !windows +// package openflow is needed by antctl which is compiled for macOS too. + +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "net" + + "github.com/vmware-tanzu/antrea/pkg/agent/openflow/cookie" + binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" +) + +// externalFlows returns the flows needed to enable SNAT for external traffic. +func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow { + if !c.enableEgress { + return nil + } + return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT) +} diff --git a/pkg/agent/openflow/pipeline_windows.go b/pkg/agent/openflow/pipeline_windows.go new file mode 100644 index 00000000000..3eba30c794b --- /dev/null +++ b/pkg/agent/openflow/pipeline_windows.go @@ -0,0 +1,253 @@ +// +build windows + +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "net" + + "github.com/vmware-tanzu/antrea/pkg/agent/config" + "github.com/vmware-tanzu/antrea/pkg/agent/openflow/cookie" + binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" +) + +const ( + markTrafficFromBridge = 5 + + // ctZoneSNAT is only used on Windows and only when AntreaProxy is enabled. + // When a Pod access a ClusterIP Service, and the IP of the selected endpoint + // is not in "cluster-cidr". The request packets need to be SNAT'd(set src IP to local Node IP) + // after have been DNAT'd(set dst IP to endpoint IP). + // For example, the endpoint Pod may run in hostNetwork mode and the IP of the endpoint + // will is the current Node IP. + // We need to use a different ct_zone to track the SNAT'd connection because OVS + // does not support doing both DNAT and SNAT in the same ct_zone. + // + // An example of the connection is a Pod accesses kubernetes API service: + // Pod --> DNAT(CtZone) --> SNAT(ctZoneSNAT) --> Endpoint(API server NodeIP) + // Pod <-- unDNAT(CtZone) <-- unSNAT(ctZoneSNAT) <-- Endpoint(API server NodeIP) + ctZoneSNAT = 0xffdc + + // snatDefaultMark indicates the packet should be SNAT'd with the default + // SNAT IP (the Node IP). + snatDefaultMark = 0b1 + + // snatCTMark indicates SNAT is performed for packets of the connection. + snatCTMark = 0x40 +) + +var ( + // snatMarkRange takes the 17th bit of register marksReg to indicate if + // the packet needs to be SNATed with Node's IP or not. + snatMarkRange = binding.Range{17, 17} +) + +// uplinkSNATFlows installs flows for traffic from the uplink port that help +// the SNAT implementation of the external traffic. +func (c *client) uplinkSNATFlows(localSubnet net.IPNet, category cookie.Category) []binding.Flow { + ctStateNext := dnatTable + if c.enableProxy { + ctStateNext = endpointDNATTable + } + bridgeOFPort := uint32(config.BridgeOFPort) + flows := []binding.Flow{ + // Mark the packet to indicate its destination MAC should be + // rewritten to the real MAC in the L3Forwarding table, if the + // packet is a reply to a local Pod from an external address. + c.pipeline[conntrackStateTable].BuildFlow(priorityHigh). + MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(false).MatchCTStateTrk(true). + MatchCTMark(snatCTMark, nil). + MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). + Action().LoadRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange). + Cookie(c.cookieAllocator.Request(category).Raw()). + Action().GotoTable(ctStateNext). + Done(), + // Output the non-SNAT packet to the bridge interface directly + // if it is received from the uplink interface. + c.pipeline[conntrackStateTable].BuildFlow(priorityNormal). + MatchProtocol(binding.ProtocolIP). + MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). + Action().Output(int(bridgeOFPort)). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + } + // Forward the IP packets from the uplink interface to + // conntrackTable. This is for unSNAT the traffic from the local + // Pod subnet to the external network. Non-SNAT packets will be + // output to the bridge port in conntrackStateTable. + if c.enableProxy { + // For the connection which is both applied DNAT and SNAT, the reply packtets + // are received from uplink and need to enter ctZoneSNAT first to do unSNAT. + // Pod --> DNAT(CtZone) --> SNAT(ctZoneSNAT) --> ExternalServer + // Pod <-- unDNAT(CtZone) <-- unSNAT(ctZoneSNAT) <-- ExternalServer + flows = append(flows, c.pipeline[uplinkTable].BuildFlow(priorityNormal). + MatchProtocol(binding.ProtocolIP). + MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). + Action().CT(false, conntrackTable, ctZoneSNAT).NAT().CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } else { + flows = append(flows, c.pipeline[uplinkTable].BuildFlow(priorityNormal). + MatchProtocol(binding.ProtocolIP). + MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). + Action().GotoTable(conntrackTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows +} + +// snatImplementationFlows installs flows that implement SNAT with OVS NAT. +func (c *client) snatImplementationFlows(nodeIP net.IP, category cookie.Category) []binding.Flow { + snatIPRange := &binding.IPRange{StartIP: nodeIP, EndIP: nodeIP} + l3FwdTable := c.pipeline[l3ForwardingTable] + nextTable := l3FwdTable.GetNext() + ctCommitTable := c.pipeline[conntrackCommitTable] + ccNextTable := ctCommitTable.GetNext() + flows := []binding.Flow{ + // Default to using Node IP as the SNAT IP for local Pods. + c.pipeline[snatTable].BuildFlow(priorityLow). + MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(true).MatchCTStateTrk(true). + MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). + Action().LoadRegRange(int(marksReg), snatDefaultMark, snatMarkRange). + Action().GotoTable(nextTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + // Force IP packet into the conntrack zone with SNAT. If the connection is SNATed, the reply packet should use + // Pod IP as the destination, and then is forwarded to conntrackStateTable. + c.pipeline[conntrackTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). + Action().CT(false, conntrackStateTable, CtZone).NAT().CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + // Perform SNAT with the default SNAT IP (the Node IP). A SNAT + // packet has these characteristics: 1) the ct_state is + // "+new+trk", 2) reg0[17] is set to 1; 3) ct_mark is set to + // 0x40. + ctCommitTable.BuildFlow(priorityNormal). + MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(true).MatchCTStateTrk(true).MatchCTStateDNAT(false). + MatchRegRange(int(marksReg), snatDefaultMark, snatMarkRange). + Action().CT(true, ccNextTable, CtZone). + SNAT(snatIPRange, nil). + LoadToMark(snatCTMark).CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + } + // The following flows are for both apply DNAT + SNAT for packets. + // If AntreaProxy is disabled, no DNAT happens in OVS pipeline. + if c.enableProxy { + flows = append(flows, []binding.Flow{ + // If the SNAT is needed after DNAT, mark the + // snatDefaultMark even the connection is not new, + // because this kind of packets need to enter ctZoneSNAT + // to make sure the SNAT can be applied before leaving + // the pipeline. + l3FwdTable.BuildFlow(priorityLow). + MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(false).MatchCTStateTrk(true).MatchCTStateDNAT(true). + MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}). + Action().LoadRegRange(int(marksReg), snatDefaultMark, snatMarkRange). + Action().GotoTable(nextTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + // If SNAT is needed after DNAT: + // - For new connection: commit to ctZoneSNAT + // - For existing connection: enter ctZoneSNAT to apply SNAT + ctCommitTable.BuildFlow(priorityNormal). + MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(true).MatchCTStateTrk(true).MatchCTStateDNAT(true). + MatchRegRange(int(marksReg), snatDefaultMark, snatMarkRange). + Action().CT(true, ccNextTable, ctZoneSNAT). + SNAT(snatIPRange, nil). + LoadToMark(snatCTMark).CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + ctCommitTable.BuildFlow(priorityNormal). + MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(false).MatchCTStateTrk(true).MatchCTStateDNAT(true). + MatchRegRange(int(marksReg), snatDefaultMark, snatMarkRange). + Action().CT(false, ccNextTable, ctZoneSNAT).NAT().CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + }...) + } + return flows +} + +// externalFlows returns the flows needed to enable SNAT for external traffic. +func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow { + flows := c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT) + flows = append(flows, c.uplinkSNATFlows(localSubnet, cookie.SNAT)...) + flows = append(flows, c.snatImplementationFlows(nodeIP, cookie.SNAT)...) + return flows +} + +// hostBridgeUplinkFlows generates the flows that forward traffic between the +// bridge local port and the uplink port to support the host traffic with +// outside. +func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Category) (flows []binding.Flow) { + bridgeOFPort := uint32(config.BridgeOFPort) + flows = []binding.Flow{ + c.pipeline[ClassifierTable].BuildFlow(priorityNormal). + MatchInPort(config.UplinkOFPort). + Action().LoadRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). + Action().GotoTable(uplinkTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + c.pipeline[ClassifierTable].BuildFlow(priorityNormal). + MatchInPort(config.BridgeOFPort). + Action().LoadRegRange(int(marksReg), markTrafficFromBridge, binding.Range{0, 15}). + Action().GotoTable(uplinkTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + // Output non-IP packets to the bridge port directly. IP packets + // are redirected to conntrackTable in uplinkSNATFlows() (in + // case they need unSNAT). + c.pipeline[uplinkTable].BuildFlow(priorityLow). + MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}). + Action().Output(int(bridgeOFPort)). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + // Forward the packet to conntrackTable if it enters the OVS + // pipeline from the bridge interface and is sent to the local + // Pod subnet. Mark the packet to indicate its destination MAC + // should be rewritten to the real MAC in the L3Frowarding + // table. This is for the case a Pod accesses a NodePort Service + // using the local Node's IP, and then the return traffic after + // the kube-proxy processing will enter the bridge from the + // bridge interface (but not the gateway interface. This is + // probably because we do not enable IP forwarding on the bridge + // interface). + c.pipeline[uplinkTable].BuildFlow(priorityHigh). + MatchProtocol(binding.ProtocolIP). + MatchRegRange(int(marksReg), markTrafficFromBridge, binding.Range{0, 15}). + MatchDstIPNet(localSubnet). + Action().LoadRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange). + Action().GotoTable(conntrackTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + // Output other packets from the bridge port to the uplink port + // directly. + c.pipeline[uplinkTable].BuildFlow(priorityLow). + MatchRegRange(int(marksReg), markTrafficFromBridge, binding.Range{0, 15}). + Action().Output(config.UplinkOFPort). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + } + return flows +} diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 51449ac4ea7..39059e30c59 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -110,7 +110,7 @@ func TestConnectivityFlows(t *testing.T) { antrearuntime.WindowsOS = runtime.GOOS } - c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false, true) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) defer func() { @@ -137,7 +137,7 @@ func TestConnectivityFlows(t *testing.T) { } func TestReplayFlowsConnectivityFlows(t *testing.T) { - c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -164,7 +164,7 @@ func TestReplayFlowsConnectivityFlows(t *testing.T) { } func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { - c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -217,12 +217,18 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { func testExternalFlows(t *testing.T, config *testConfig) { nodeIP := config.nodeConfig.NodeIPAddr.IP - localSubnet := config.nodeConfig.PodIPv4CIDR + var localSubnet *net.IPNet + if config.nodeConfig.PodIPv4CIDR != nil { + localSubnet = config.nodeConfig.PodIPv4CIDR + } else { + localSubnet = config.nodeConfig.PodIPv6CIDR + } + gwMAC := config.nodeConfig.GatewayConfig.MAC if err := c.InstallExternalFlows(); err != nil { t.Errorf("Failed to install OpenFlow entries to allow Pod to communicate to the external addresses: %v", err) } - for _, tableFlow := range prepareExternalFlows(nodeIP, localSubnet, config.globalMAC) { + for _, tableFlow := range prepareExternalFlows(nodeIP, localSubnet, gwMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } @@ -337,7 +343,7 @@ func TestNetworkPolicyFlows(t *testing.T) { // Initialize ovs metrics (Prometheus) to test them metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -447,7 +453,7 @@ func TestIPv6ConnectivityFlows(t *testing.T) { // Initialize ovs metrics (Prometheus) to test them metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false, true) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -465,6 +471,7 @@ func TestIPv6ConnectivityFlows(t *testing.T) { testInstallGatewayFlows, testUninstallPodFlows, testUninstallNodeFlows, + testExternalFlows, } { f(t, config) } @@ -478,7 +485,7 @@ type svcConfig struct { } func TestProxyServiceFlows(t *testing.T) { - c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, ovsconfig.OVSDatapathNetdev, true, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -879,6 +886,8 @@ func prepareConfiguration() *testConfig { func prepareIPv6Configuration() *testConfig { podMAC, _ := net.ParseMAC("aa:aa:aa:aa:aa:13") + nodeIP, nodeSubnet, _ := net.ParseCIDR("a963:ca9b:172:10::11/64") + nodeSubnet.IP = nodeIP gwMAC, _ := net.ParseMAC("aa:aa:aa:aa:aa:11") gatewayConfig := &config1.GatewayConfig{ @@ -886,6 +895,7 @@ func prepareIPv6Configuration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ + NodeIPAddr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv6CIDR: podIPv6CIDR, } @@ -980,7 +990,7 @@ func preparePodFlows(podIPs []net.IP, podMAC net.HardwareAddr, podOFPort uint32, []*ofTestUtils.ExpectFlow{ { MatchStr: fmt.Sprintf("priority=200,%s,reg0=0x80000/0x80000,%s=%s", ipProto, nwDstField, podIP.String()), - ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,goto_table:71", gwMAC.String(), podMAC.String()), + ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,goto_table:72", gwMAC.String(), podMAC.String()), }, }, }, @@ -1118,7 +1128,7 @@ func prepareNodeFlows(peerSubnet net.IPNet, peerGwIP, peerNodeIP net.IP, vMAC, l []*ofTestUtils.ExpectFlow{ { MatchStr: fmt.Sprintf("priority=200,%s,%s=%s", ipProtoStr, nwDstFieldName, peerSubnet.String()), - ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,set_field:%s->tun_dst,goto_table:71", localGwMAC.String(), vMAC.String(), peerNodeIP.String()), + ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,set_field:%s->tun_dst,goto_table:72", localGwMAC.String(), vMAC.String(), peerNodeIP.String()), }, }, }) @@ -1149,8 +1159,8 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { tableID: 105, flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:106"}}, } - table71Flows := expectTableFlows{ - tableID: 71, + table72Flows := expectTableFlows{ + tableID: 72, flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:80"}}, } if config.enableIPv4 { @@ -1162,7 +1172,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ip,reg0=0x1/0xffff", ActStr: "ct(commit,table=106,zone=65520,exec(load:0x20->NXM_NX_CT_MARK[])"}, &ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ip", ActStr: "ct(commit,table=106,zone=65520)"}, ) - table71Flows.flows = append(table71Flows.flows, + table72Flows.flows = append(table72Flows.flows, &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ip,reg0=0x1/0xffff", ActStr: "goto_table:80"}, &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ip", ActStr: "dec_ttl,goto_table:80"}, ) @@ -1176,13 +1186,13 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ipv6,reg0=0x1/0xffff", ActStr: "ct(commit,table=106,zone=65510,exec(load:0x20->NXM_NX_CT_MARK[])"}, &ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ipv6", ActStr: "ct(commit,table=106,zone=65510)"}, ) - table71Flows.flows = append(table71Flows.flows, + table72Flows.flows = append(table72Flows.flows, &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ipv6,reg0=0x1/0xffff", ActStr: "goto_table:80"}, &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ipv6", ActStr: "dec_ttl,goto_table:80"}, ) } return []expectTableFlows{ - table31Flows, table105Flows, table71Flows, + table31Flows, table105Flows, table72Flows, { uint8(0), []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "drop"}}, @@ -1267,72 +1277,52 @@ func prepareIPNetAddresses(addresses []string) []types.Address { return ipAddresses } -func prepareExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, vMAC net.HardwareAddr) []expectTableFlows { +func prepareExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, gwMAC net.HardwareAddr) []expectTableFlows { + var ipProtoStr, nwDstFieldName string + if localSubnet.IP.To4() != nil { + ipProtoStr = "ip" + nwDstFieldName = "nw_dst" + } else { + ipProtoStr = "ipv6" + nwDstFieldName = "ipv6_dst" + } return []expectTableFlows{ { - uint8(5), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: fmt.Sprintf("priority=200,ip,reg0=0x4/0xffff"), - ActStr: "ct(table=30,zone=65500,nat)", - }, - }, - }, - { - uint8(30), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: "priority=200,ip", ActStr: "ct(table=31,zone=65520,nat)", - }, - }, - }, - { - uint8(31), + // snatCommonFlows() + uint8(70), []*ofTestUtils.ExpectFlow{ { - MatchStr: "priority=210,ct_state=-new+trk,ct_mark=0x40,ip,reg0=0x4/0xffff", - ActStr: "load:0x1->NXM_NX_REG0[19],goto_table:42", - }, - { - MatchStr: fmt.Sprintf("priority=200,ip,reg0=0x4/0xffff"), - ActStr: "LOCAL", + MatchStr: fmt.Sprintf("priority=200,%s,reg0=0x2/0xffff,%s=%s", ipProtoStr, nwDstFieldName, localSubnet.String()), + ActStr: "goto_table:80", }, - }, - }, - { - uint8(70), - []*ofTestUtils.ExpectFlow{ { - MatchStr: fmt.Sprintf("priority=200,ip,reg0=0x2/0xffff,nw_dst=%s", localSubnet.String()), + MatchStr: fmt.Sprintf("priority=200,%s,reg0=0x2/0xffff,%s=%s", ipProtoStr, nwDstFieldName, nodeIP.String()), ActStr: "goto_table:80", }, { - MatchStr: fmt.Sprintf("priority=200,ip,reg0=0x2/0xffff,nw_dst=%s", nodeIP.String()), + MatchStr: fmt.Sprintf("priority=200,ct_mark=0x20,%s,reg0=0x2/0xffff", ipProtoStr), ActStr: "goto_table:80", }, { - MatchStr: "priority=200,ct_mark=0x20,ip,reg0=0x2/0xffff", ActStr: "goto_table:80", + MatchStr: fmt.Sprintf("priority=190,%s,reg0=0x2/0xffff", ipProtoStr), + ActStr: "goto_table:71", }, { - MatchStr: "priority=190,ct_state=+new+trk,ip,reg0=0x2/0xffff", - ActStr: "load:0x1->NXM_NX_REG0[17],goto_table:80", + MatchStr: fmt.Sprintf("priority=190,%s,reg0=0/0xffff", ipProtoStr), + ActStr: fmt.Sprintf("set_field:%s->eth_dst,goto_table:71", gwMAC.String()), }, }, }, { - uint8(105), + uint8(71), []*ofTestUtils.ExpectFlow{ { - MatchStr: "priority=200,ct_state=+new+trk-dnat,ip,reg0=0x20000/0x20000", - ActStr: fmt.Sprintf("ct(commit,table=110,zone=65520,nat(src=%s),exec(load:0x40->NXM_NX_CT_MARK[]))", nodeIP.String()), - }, - { - MatchStr: "priority=200,ct_state=+new+trk+dnat,ip,reg0=0x20000/0x20000", - ActStr: fmt.Sprintf("ct(commit,table=110,zone=65500,nat(src=%s),exec(load:0x40->NXM_NX_CT_MARK[]))", nodeIP.String()), + MatchStr: fmt.Sprintf("priority=190,ct_state=+new+trk,%s,reg0=0/0xffff", ipProtoStr), + ActStr: "drop", }, { - MatchStr: "priority=200,ct_state=-new+trk+dnat,ip,reg0=0x20000/0x20000", - ActStr: "ct(table=110,zone=65500,nat)", + MatchStr: "priority=0", + ActStr: "goto_table:80", }, }, },