diff --git a/pkg/agent/config/service_config.go b/pkg/agent/config/service_config.go new file mode 100644 index 00000000000..b5b797e6490 --- /dev/null +++ b/pkg/agent/config/service_config.go @@ -0,0 +1,21 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import v1 "k8s.io/api/core/v1" + +const ( + ServiceTypeExternalIP v1.ServiceType = "ExternalIP" +) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index e49db455199..baf15dad789 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -2334,9 +2334,9 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, case binding.ProtocolSCTPv6: learnFlowBuilderLearnAction = learnFlowBuilderLearnAction.MatchLearnedSCTPv6DstPort() } - // If externalTrafficPolicy of NodePort/LoadBalancer is Cluster, the learned flow which - // is used to match the first packet of NodePort/LoadBalancer also requires SNAT. - if (svcType == v1.ServiceTypeNodePort || svcType == v1.ServiceTypeLoadBalancer) && !nodeLocalExternal { + // If externalTrafficPolicy of NodePort/LoadBalancer/ExternalIP is Cluster, the learned flow which + // is used to match the first packet of NodePort/LoadBalancer/ExternalIP also requires SNAT. + if (svcType == v1.ServiceTypeNodePort || svcType == v1.ServiceTypeLoadBalancer || svcType == config.ServiceTypeExternalIP) && !nodeLocalExternal { learnFlowBuilderLearnAction = learnFlowBuilderLearnAction.LoadRegMark(ToClusterServiceRegMark) } @@ -2400,10 +2400,10 @@ func (f *featureService) serviceLBFlow(groupID binding.GroupIDType, MatchDstPort(svcPort, nil). Action().LoadRegMark(regMarksToLoad...) } else { - // If externalTrafficPolicy of a LoadBalancer Service is Cluster (nodeLocalExternal=false), it loads + // If externalTrafficPolicy of a LoadBalancer/ExternalIP Service is Cluster (nodeLocalExternal=false), it loads // ToClusterServiceRegMark to indicate it. The mark will be checked later when determining if SNAT is required // or not. - if serviceType == v1.ServiceTypeLoadBalancer && !nodeLocalExternal { + if (serviceType == v1.ServiceTypeLoadBalancer || serviceType == config.ServiceTypeExternalIP) && !nodeLocalExternal { regMarksToLoad = append(regMarksToLoad, ToClusterServiceRegMark) } flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityNormal). diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 1774915d96d..760eba5a679 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -189,12 +189,18 @@ func (p *proxier) removeServiceFlows(svcInfo *types.ServiceInfo) bool { klog.ErrorS(err, "Error when uninstalling ClusterIP flows for Service", "ServiceInfo", svcInfoStr) return false } - // Remove NodePort flows and configurations. + if p.proxyAll { + // Remove NodePort flows and configurations. if err := p.uninstallNodePortService(uint16(svcInfo.NodePort()), svcInfo.OFProtocol); err != nil { klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } + // Remove ExternalIP flows and configurations. + if err := p.uninstallExternalIPService(svcInfoStr, svcInfo.ExternalIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } // Remove LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs { @@ -325,7 +331,9 @@ func serviceIdentityChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { } func serviceExternalAddressesChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { - return svcInfo.NodePort() != pSvcInfo.NodePort() || !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) + return svcInfo.NodePort() != pSvcInfo.NodePort() || + !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) || + !slices.Equal(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) } // smallSliceDifference builds a slice which includes all the strings from s1 @@ -383,6 +391,36 @@ func (p *proxier) uninstallNodePortService(svcPort uint16, protocol binding.Prot return nil } +func (p *proxier) installExternalIPService(svcInfoStr string, groupID binding.GroupIDType, externalIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { + for _, externalIP := range externalIPStrings { + if externalIP != "" { + ip := net.ParseIP(externalIP) + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, nodeLocalExternal, agentconfig.ServiceTypeExternalIP, false); err != nil { + return fmt.Errorf("failed to install ExternalIP load balancing flows: %w", err) + } + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddExternalIP); err != nil { + return fmt.Errorf("failed to install ExternalIP traffic redirecting routes: %w", err) + } + } + } + return nil +} + +func (p *proxier) uninstallExternalIPService(svcInfoStr string, externalIPStrings []string, svcPort uint16, protocol binding.Protocol) error { + for _, externalIP := range externalIPStrings { + if externalIP != "" { + ip := net.ParseIP(externalIP) + if err := p.ofClient.UninstallServiceFlows(ip, svcPort, protocol); err != nil { + return fmt.Errorf("failed to remove ExternalIP load balancing flows: %w", err) + } + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteExternalIP); err != nil { + return fmt.Errorf("failed to remove ExternalIP traffic redirecting routes: %w", err) + } + } + } + return nil +} + func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { @@ -588,8 +626,14 @@ func (p *proxier) installServiceFlows(svcInfo *types.ServiceInfo, internalGroupI klog.ErrorS(err, "Error when installing ClusterIP flows for Service", "ServiceInfo", svcInfoStr) return false } - // Install NodePort flows and configurations. + if p.proxyAll { + // Install ExternalIP flows and configurations. + if err := p.installExternalIPService(svcInfo.String(), externalGroupID, svcInfo.ExternalIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + klog.ErrorS(err, "Error when installing ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } + // Install NodePort flows and configurations. if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { klog.ErrorS(err, "Error when installing NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false @@ -621,6 +665,16 @@ func (p *proxier) updateServiceExternalAddresses(pSvcInfo, svcInfo *types.Servic return false } } + deletedExternalIPs := smallSliceDifference(pSvcInfo.ExternalIPStrings(), svcInfo.ExternalIPStrings()) + addedExternalIPs := smallSliceDifference(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) + if err := p.uninstallExternalIPService(pSvcInfoStr, deletedExternalIPs, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling ExternalIP flows and configurations for Service", "ServiceInfo", pSvcInfoStr) + return false + } + if err := p.installExternalIPService(svcInfoStr, externalGroupID, addedExternalIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + klog.ErrorS(err, "Error when installing ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } if p.proxyLoadBalancerIPs { deletedLoadBalancerIPs := smallSliceDifference(pSvcInfo.LoadBalancerIPStrings(), svcInfo.LoadBalancerIPStrings()) @@ -666,7 +720,7 @@ func compareEndpoints(endpointsCached map[string]k8sproxy.Endpoint, endpointsIns // endpoints or services resources are not synced. syncProxyRules is only called // through the Run method of the runner object, and all calls are serialized. // This method is the only one that changes internal state, but -// GetServiceFlowKeys(), which is called by the the "/ovsflows" API handler, +// GetServiceFlowKeys(), which is called by the "/ovsflows" API handler, // also reads service and endpoints maps, so serviceEndpointsMapsMutex is used // to protect these two maps. func (p *proxier) syncProxyRules() { diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index 5359a1e3ca7..1ce30a9bc1b 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -61,6 +61,8 @@ var ( loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") svcNodePortIPv4 = net.ParseIP("192.168.77.100") svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") + externalIPv4 = net.ParseIP("192.168.77.101") + externalIPv6 = net.ParseIP("2001::192:168:77:101") nodePortAddressesIPv4 = []net.IP{svcNodePortIPv4} nodePortAddressesIPv6 = []net.IP{svcNodePortIPv6} @@ -139,6 +141,7 @@ func makeTestEndpointSlice(namespace, name string, eps []discovery.Endpoint, por func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs []net.IP, svcPort int32, protocol corev1.Protocol, affinitySeconds *int32, @@ -151,6 +154,11 @@ func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, Port: svcPort, Protocol: protocol, }} + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } if internalTrafficPolicy != nil { svc.Spec.InternalTrafficPolicy = internalTrafficPolicy } @@ -170,6 +178,7 @@ func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs []net.IP, svcPort, svcNodePort int32, protocol corev1.Protocol, @@ -185,6 +194,11 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, Port: svcPort, Protocol: protocol, }} + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy if affinitySeconds != nil { @@ -200,6 +214,7 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs, loadBalancerIPs []net.IP, svcPort, svcNodePort int32, @@ -212,9 +227,16 @@ func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, svc.Spec.Type = corev1.ServiceTypeLoadBalancer var ingress []corev1.LoadBalancerIngress for _, ip := range loadBalancerIPs { - ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + if ip != nil { + ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + } } svc.Status.LoadBalancer.Ingress = ingress + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } svc.Spec.Ports = []corev1.ServicePort{{ NodePort: svcNodePort, Name: svcPortName.Port, @@ -355,6 +377,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP func testClusterIPAdd(t *testing.T, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, isIPv6 bool, @@ -377,7 +400,11 @@ func testClusterIPAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true)) + var externalIPs []net.IP + if externalIP != nil { + externalIPs = append(externalIPs, externalIP) + } + allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, externalIPs, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true)) makeServiceMap(fp, allSvcs...) if !endpointSliceEnabled { @@ -402,8 +429,9 @@ func testClusterIPAdd(t *testing.T, nodeName = hostname serving = true } - expectedAllEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} - if !nodeLocalInternal { + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} + expectedAllEps := expectedLocalEps + if !nodeLocalInternal || externalIP != nil { expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, serving, false, nil)) } @@ -412,11 +440,28 @@ func testClusterIPAdd(t *testing.T, bindingProtocol = binding.ProtocolTCPv6 } - groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) - mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) - + if nodeLocalInternal == false { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), false, agentconfig.ServiceTypeExternalIP, false).Times(1) + } + } else { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedLocalEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) + if externalIP != nil { + groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), false, agentconfig.ServiceTypeExternalIP, false).Times(1) + } + } + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIP(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) assert.Contains(t, fp.endpointsInstalledMap, svcPortName) @@ -425,6 +470,7 @@ func testClusterIPAdd(t *testing.T, func testLoadBalancerAdd(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, loadBalancerIP net.IP, @@ -454,7 +500,9 @@ func testLoadBalancerAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + []net.IP{externalIP}, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -518,6 +566,9 @@ func testLoadBalancerAdd(t *testing.T, if proxyLoadBalancerIPs { mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false).Times(1) } + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, agentconfig.ServiceTypeExternalIP, false).Times(1) + } } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) @@ -527,6 +578,9 @@ func testLoadBalancerAdd(t *testing.T, if proxyLoadBalancerIPs { mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false).Times(1) } + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, agentconfig.ServiceTypeExternalIP, false).Times(1) + } groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } @@ -534,6 +588,9 @@ func testLoadBalancerAdd(t *testing.T, mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIP(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -543,6 +600,7 @@ func testLoadBalancerAdd(t *testing.T, func testNodePortAdd(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, isIPv6 bool, @@ -567,7 +625,9 @@ func testNodePortAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -628,17 +688,25 @@ func testNodePortAdd(t *testing.T, groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, agentconfig.ServiceTypeExternalIP, false).Times(1) + } } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) - + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, agentconfig.ServiceTypeExternalIP, false).Times(1) + } groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIP(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -649,36 +717,36 @@ func TestClusterIPAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) }) }) @@ -688,72 +756,72 @@ func TestLoadBalancerAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) }) }) }) @@ -917,60 +985,60 @@ func TestNodePortAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, false, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, false, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, false, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, false, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, true, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, false, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, false, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, false, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, false, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, true, true) }) }) }) @@ -988,8 +1056,8 @@ func TestClusterSkipServices(t *testing.T) { skippedServiceName := strings.Split(skippedServiceNN, "/")[1] svc1PortName := makeSvcPortName(skippedServiceNamespace, skippedServiceName, strconv.Itoa(svc1Port), corev1.ProtocolTCP) svc2PortName := makeSvcPortName("kube-system", "test", strconv.Itoa(svc2Port), corev1.ProtocolTCP) - svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, int32(svc1Port), corev1.ProtocolTCP, nil, nil, false) - svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, int32(svc2Port), corev1.ProtocolTCP, nil, nil, false) + svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, nil, int32(svc1Port), corev1.ProtocolTCP, nil, nil, false) + svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, nil, int32(svc2Port), corev1.ProtocolTCP, nil, nil, false) svcs := []*corev1.Service{svc1, svc2} epSubset := makeTestEndpointSubset(&svc1PortName, ep1IP, int32(svc1Port), corev1.ProtocolTCP, false) @@ -998,7 +1066,7 @@ func TestClusterSkipServices(t *testing.T) { ep2 := makeTestEndpoints(&svc2PortName, []corev1.EndpointSubset{*epSubset}) eps := []*corev1.Endpoints{ep1, ep2} - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) + testClusterIPAdd(t, svc1IPv4, nil, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) } func TestDualStackService(t *testing.T) { @@ -1050,7 +1118,7 @@ func TestDualStackService(t *testing.T) { assert.Contains(t, fpv6.serviceInstalledMap, svcPortName) } -func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, nodeLocalInternal, endpointSliceEnabled bool) { +func testClusterIPRemove(t *testing.T, svcIP, externalIP, epIP net.IP, isIPv6 bool, nodeLocalInternal, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1065,7 +1133,7 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true) + svc := makeTestClusterIPService(&svcPortName, svcIP, []net.IP{externalIP}, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true) makeServiceMap(fp, svc) var ep *corev1.Endpoints @@ -1085,17 +1153,39 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n bindingProtocol = binding.ProtocolTCPv6 } - groupID := groupAllocator.Next() - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) - if !nodeLocalInternal { + if nodeLocalInternal == false { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - } - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) - mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - if !nodeLocalInternal { + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), false, agentconfig.ServiceTypeExternalIP, false).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + } + } else { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + if externalIP != nil { + groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), false, agentconfig.ServiceTypeExternalIP, false).Times(1) + + mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) + mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + } + } + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIP(externalIP) + mockRouteClient.EXPECT().DeleteExternalIP(externalIP) } - mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1115,7 +1205,7 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n assert.False(t, exists) } -func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { +func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP, externalIP, epIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1126,7 +1216,9 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, } fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -1162,12 +1254,20 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, agentconfig.ServiceTypeExternalIP, false).Times(1) + mockRouteClient.EXPECT().AddExternalIP(externalIP) + } mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIP(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1185,7 +1285,7 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, assert.NotContains(t, fp.endpointsInstalledMap, svcPortName) } -func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, loadBalancerIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { +func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP, externalIP, epIP, loadBalancerIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1199,7 +1299,9 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + []net.IP{externalIP}, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -1236,9 +1338,12 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, agentconfig.ServiceTypeExternalIP, false).Times(1) + mockRouteClient.EXPECT().AddExternalIP(externalIP) + } mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) @@ -1247,6 +1352,10 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIP(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1268,36 +1377,36 @@ func TestClusterIPRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, false, false) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, false, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, true, false) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, false, true) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, false, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, true, true) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, true, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, false, false) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, false, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, true, false) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, false, true) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, false, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, true, true) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, true, true) }) }) }) @@ -1306,18 +1415,18 @@ func TestClusterIPRemove(t *testing.T) { func TestNodePortRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false, false) + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, false, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false, true) + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, false, true) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true, false) + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, true, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true, true) + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, true, true) }) }) } @@ -1325,18 +1434,18 @@ func TestNodePortRemove(t *testing.T) { func TestLoadBalancerRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false, false) + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, loadBalancerIPv4, false, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false, true) + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, loadBalancerIPv4, false, true) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true, false) + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, loadBalancerIPv6, true, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true, true) + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, loadBalancerIPv6, true, true) }) }) @@ -1349,8 +1458,8 @@ func testClusterIPNoEndpoint(t *testing.T, svcIP net.IP, isIPv6 bool) { groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc) makeEndpointSliceMap(fp) @@ -1382,14 +1491,18 @@ func testNodePortNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) - updatedSvc := makeTestNodePortService(&svcPortName, svcIP, + updatedSvc := makeTestNodePortService(&svcPortName, + svcIP, + nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, @@ -1442,7 +1555,9 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -1450,7 +1565,9 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP nil, &internalTrafficPolicy, externalTrafficPolicy) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, + updatedSvc := makeTestLoadBalancerService(&svcPortName, + svcIP, + nil, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), @@ -1510,8 +1627,8 @@ func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP svcPortNameTCP := makeSvcPortName("ns", "svc-tcp", strconv.Itoa(svcPort), corev1.ProtocolTCP) svcPortNameUDP := makeSvcPortName("ns", "svc-udp", strconv.Itoa(svcPort), corev1.ProtocolUDP) - svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, int32(svcPort), corev1.ProtocolUDP, nil, nil, false) + svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, nil, int32(svcPort), corev1.ProtocolUDP, nil, nil, false) makeServiceMap(fp, svcTCP, svcUDP) epTCP, epPortTCP := makeTestEndpointSliceEndpointAndPort(&svcPortNameTCP, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -1565,7 +1682,7 @@ func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -1732,14 +1849,14 @@ func testServiceClusterIPUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1834,14 +1951,14 @@ func testServicePortUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1937,11 +2054,11 @@ func testServiceNodePortUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2023,11 +2140,11 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) } makeServiceMap(fp, svc) @@ -2130,8 +2247,8 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, internalTrafficPolicyCluster := corev1.ServiceInternalTrafficPolicyCluster internalTrafficPolicyLocal := corev1.ServiceInternalTrafficPolicyLocal - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster, false) - updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster, false) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal, false) makeServiceMap(fp, svc) remoteEp, remoteEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) @@ -2222,8 +2339,8 @@ func testServiceIngressIPsUpdate(t *testing.T, updatedLoadBalancerIPStrs = append(updatedLoadBalancerIPStrs, ip.String()) } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -2307,14 +2424,14 @@ func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, updatedAffinitySeconds := int32(100) switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2406,14 +2523,14 @@ func testServiceSessionAffinityTypeUpdate(t *testing.T, affinitySeconds := int32(100) switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2501,8 +2618,8 @@ func TestServicesWithSameEndpoints(t *testing.T) { svcPortName1 := makeSvcPortName("ns", "svc1", strconv.Itoa(svcPort), corev1.ProtocolTCP) svcPortName2 := makeSvcPortName("ns", "svc2", strconv.Itoa(svcPort), corev1.ProtocolTCP) - svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc1, svc2) ep1, ep1Port := makeTestEndpointSliceEndpointAndPort(&svcPortName1, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) @@ -2569,7 +2686,7 @@ func TestMetrics(t *testing.T) { servicesInstallMetric = metrics.ServicesInstalledTotalV6.GaugeMetric } - testClusterIPAdd(t, tc.svcIP, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, tc.svcIP, nil, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) v, err := testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) assert.Equal(t, 0, int(v)) @@ -2583,7 +2700,7 @@ func TestMetrics(t *testing.T) { assert.Equal(t, 2, int(v)) assert.NoError(t, err) - testClusterIPRemove(t, tc.svcIP, tc.ep1IP, tc.isIPv6, false, false) + testClusterIPRemove(t, tc.svcIP, nil, tc.ep1IP, tc.isIPv6, false, false) v, err = testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) @@ -2608,6 +2725,7 @@ func TestGetServiceFlowKeys(t *testing.T) { groupAllocator := openflow.NewGroupAllocator(false) svc := makeTestNodePortService(&svcPortName, svc1IPv4, + nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, diff --git a/pkg/agent/route/interfaces.go b/pkg/agent/route/interfaces.go index 8975c98f6bd..4832e3a8995 100644 --- a/pkg/agent/route/interfaces.go +++ b/pkg/agent/route/interfaces.go @@ -71,6 +71,12 @@ type Interface interface { // DeleteLoadBalancer deletes related configurations when a LoadBalancer IP is deleted. DeleteLoadBalancer(externalIP net.IP) error + // AddExternalIP adds configurations when an external IP is added. + AddExternalIP(externalIP net.IP) error + + // DeleteExternalIP deletes related configurations when an external IP is deleted. + DeleteExternalIP(svcIP net.IP) error + // Run starts the sync loop. Run(stopCh <-chan struct{}) diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 51497536112..0d528e559e2 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -1408,24 +1408,11 @@ func (c *Client) addVirtualNodePortDNATIPRoute(isIPv6 bool) error { // AddLoadBalancer is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea // gateway on host. func (c *Client) AddLoadBalancer(svcIP net.IP) error { - linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - isIPv6 := utilnet.IsIPv6(svcIP) - var gw net.IP - var mask int - if !isIPv6 { - gw = config.VirtualServiceIPv4 - mask = net.IPv4len * 8 - } else { - gw = config.VirtualServiceIPv6 - mask = net.IPv6len * 8 - } - - route := generateRoute(svcIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) - if err := c.netlink.RouteReplace(route); err != nil { - return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIP.String(), err) + svcIPStr := svcIP.String() + if err := c.addServiceRoute(svcIP); err != nil { + return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIPStr, err) } - klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Store(svcIP.String(), route) + klog.V(4).InfoS("Added route for LoadBalancer ingress IP", "IP", svcIPStr) return nil } @@ -1434,21 +1421,30 @@ func (c *Client) AddLoadBalancer(svcIP net.IP) error { // gateway on host. func (c *Client) DeleteLoadBalancer(svcIP net.IP) error { svcIPStr := svcIP.String() - route, found := c.serviceRoutes.Load(svcIPStr) - if !found { - klog.V(2).InfoS("Didn't find route for LoadBalancer ingress IP", "ingressIP", svcIPStr) - return nil + if err := c.deleteService(svcIP); err != nil { + return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) } - if err := c.netlink.RouteDel(route.(*netlink.Route)); err != nil { - if err.Error() == "no such process" { - klog.InfoS("Failed to delete route for LoadBalancer ingress IP since it doesn't exist", "route", route) - } else { - return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) - } + klog.V(4).InfoS("Deleted route for LoadBalancer ingress IP", "IP", svcIPStr) + return nil +} + +// AddExternalIP is used to add routing entry which is used to route ExternalIP to Antrea gateway on host. +func (c *Client) AddExternalIP(svcIP net.IP) error { + svcIPStr := svcIP.String() + if err := c.addServiceRoute(svcIP); err != nil { + return fmt.Errorf("failed to install route for ExternalIP %s: %w", svcIPStr, err) } - klog.V(4).InfoS("Deleted LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Delete(svcIPStr) + klog.V(4).InfoS("Added route for ExternalIP", "IP", svcIPStr) + return nil +} +// DeleteExternalIP is used to delete routing entry which is used to route ExternalIP to Antrea gateway on host. +func (c *Client) DeleteExternalIP(svcIP net.IP) error { + svcIPStr := svcIP.String() + if err := c.deleteService(svcIP); err != nil { + return fmt.Errorf("failed to delete route for ExternalIP %s: %w", svcIPStr, err) + } + klog.V(4).InfoS("Deleted route for ExternalIP", "IP", svcIPStr) return nil } @@ -1600,3 +1596,38 @@ func generateNeigh(ip net.IP, linkIndex int) *netlink.Neigh { HardwareAddr: globalVMAC, } } + +func (c *Client) addServiceRoute(svcIP net.IP) error { + linkIndex := c.nodeConfig.GatewayConfig.LinkIndex + isIPv6 := utilnet.IsIPv6(svcIP) + var gw net.IP + var mask int + if !isIPv6 { + gw = config.VirtualServiceIPv4 + mask = net.IPv4len * 8 + } else { + gw = config.VirtualServiceIPv6 + mask = net.IPv6len * 8 + } + + route := generateRoute(svcIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) + return c.netlink.RouteReplace(route) +} + +func (c *Client) deleteService(svcIP net.IP) error { + svcIPStr := svcIP.String() + route, found := c.serviceRoutes.Load(svcIPStr) + if !found { + klog.V(2).InfoS("Didn't find route for Service IP", "IP", svcIPStr) + return nil + } + if err := c.netlink.RouteDel(route.(*netlink.Route)); err != nil { + if err.Error() == "no such process" { + klog.InfoS("Failed to delete route for Service IP since it doesn't exist", "IP", svcIPStr) + } else { + return err + } + } + c.serviceRoutes.Delete(svcIPStr) + return nil +} diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 82257b9a995..67876ceca73 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -510,40 +510,41 @@ func (c *Client) DeleteNodePort(nodePortAddresses []net.IP, port uint16, protoco return nil } -func (c *Client) AddLoadBalancer(externalIP net.IP) error { - svcIPStr := externalIP.String() - linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - gw := config.VirtualServiceIPv4 - metric := util.MetricHigh - _, svcIPNet, _ := net.ParseCIDR(svcIPStr) - - route := generateRoute(svcIPNet, gw, linkIndex, metric) - if err := util.ReplaceNetRoute(route); err != nil { +func (c *Client) AddLoadBalancer(svcIP net.IP) error { + svcIPStr := svcIP.String() + if err := c.addServiceRoute(svcIP); err != nil { return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIPStr, err) } - klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Store(svcIPStr, route) + klog.V(4).InfoS("Added route for LoadBalancer ingress IP", "IP", svcIPStr) return nil } - -func (c *Client) DeleteLoadBalancer(externalIP net.IP) error { - svcIPStr := externalIP.String() - route, found := c.serviceRoutes.Load(svcIPStr) - if !found { - klog.V(2).InfoS("Didn't find route for LoadBalancer ingress IP", "ingressIP", svcIPStr) - return nil +func (c *Client) DeleteLoadBalancer(svcIP net.IP) error { + svcIPStr := svcIP.String() + if err := c.deleteService(svcIP); err != nil { + return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) } - if err := util.RemoveNetRoute(route.(*util.Route)); err != nil { - if strings.Contains(err.Error(), "No matching MSFT_NetRoute objects") { - klog.InfoS("Failed to delete route for LoadBalancer ingress IP since it doesn't exist", "route", route) - } else { - return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) - } + klog.V(4).InfoS("Deleted route for LoadBalancer ingress IP", "IP", svcIPStr) + return nil +} + +// AddExternalIP is used to add routing entry which is used to route ExternalIP to Antrea gateway on host. +func (c *Client) AddExternalIP(svcIP net.IP) error { + svcIPStr := svcIP.String() + if err := c.addServiceRoute(svcIP); err != nil { + return fmt.Errorf("failed to install route for ExternalIP %s: %w", svcIPStr, err) } - klog.V(4).InfoS("Deleted LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Delete(svcIPStr) + klog.V(4).InfoS("Added route for ExternalIP", "IP", svcIPStr) + return nil +} +// DeleteExternalIP is used to delete routing entry which is used to route ExternalIP to Antrea gateway on host. +func (c *Client) DeleteExternalIP(svcIP net.IP) error { + svcIPStr := svcIP.String() + if err := c.deleteService(svcIP); err != nil { + return fmt.Errorf("failed to delete route for ExternalIP %s: %w", svcIPStr, err) + } + klog.V(4).InfoS("Deleted route for ExternalIP", "IP", svcIPStr) return nil } @@ -572,3 +573,36 @@ func generateNeigh(ip net.IP, linkIndex int) *util.Neighbor { State: "Permanent", } } + +func (c *Client) addServiceRoute(svcIP net.IP) error { + svcIPStr := svcIP.String() + linkIndex := c.nodeConfig.GatewayConfig.LinkIndex + gw := config.VirtualServiceIPv4 + metric := util.MetricHigh + _, svcIPNet, _ := net.ParseCIDR(svcIPStr) + + route := generateRoute(svcIPNet, gw, linkIndex, metric) + if err := util.ReplaceNetRoute(route); err != nil { + return err + } + c.serviceRoutes.Store(svcIPStr, route) + return nil +} + +func (c *Client) deleteService(svcIP net.IP) error { + svcIPStr := svcIP.String() + route, found := c.serviceRoutes.Load(svcIPStr) + if !found { + klog.V(2).InfoS("Didn't find route for Service IP", "IP", svcIPStr) + return nil + } + if err := util.RemoveNetRoute(route.(*util.Route)); err != nil { + if strings.Contains(err.Error(), "No matching MSFT_NetRoute objects") { + klog.InfoS("Failed to delete route for Service IP since it doesn't exist", "IP", svcIPStr) + } else { + return err + } + } + c.serviceRoutes.Delete(svcIPStr) + return nil +} diff --git a/pkg/agent/route/testing/mock_route.go b/pkg/agent/route/testing/mock_route.go index 8d5a3256530..b6920bb11bb 100644 --- a/pkg/agent/route/testing/mock_route.go +++ b/pkg/agent/route/testing/mock_route.go @@ -50,6 +50,20 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { return m.recorder } +// AddExternalIP mocks base method +func (m *MockInterface) AddExternalIP(arg0 net.IP) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddExternalIP", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddExternalIP indicates an expected call of AddExternalIP +func (mr *MockInterfaceMockRecorder) AddExternalIP(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddExternalIP", reflect.TypeOf((*MockInterface)(nil).AddExternalIP), arg0) +} + // AddLoadBalancer mocks base method func (m *MockInterface) AddLoadBalancer(arg0 net.IP) error { m.ctrl.T.Helper() @@ -120,6 +134,20 @@ func (mr *MockInterfaceMockRecorder) AddSNATRule(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddSNATRule", reflect.TypeOf((*MockInterface)(nil).AddSNATRule), arg0, arg1) } +// DeleteExternalIP mocks base method +func (m *MockInterface) DeleteExternalIP(arg0 net.IP) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteExternalIP", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteExternalIP indicates an expected call of DeleteExternalIP +func (mr *MockInterfaceMockRecorder) DeleteExternalIP(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExternalIP", reflect.TypeOf((*MockInterface)(nil).DeleteExternalIP), arg0) +} + // DeleteLoadBalancer mocks base method func (m *MockInterface) DeleteLoadBalancer(arg0 net.IP) error { m.ctrl.T.Helper()