fix: generating endpoint locally when CNS is not reachabe.

Author: behzad-mir

cni/network/network.go

@@ -1070,49 +1070,11 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
 	}
 	logger.Info("Retrieved network info, populating endpoint infos with container id", zap.String("containerID", args.ContainerID))
 
-	var epInfos []*network.EndpointInfo
-	if plugin.nm.IsStatelessCNIMode() {
-		// network ID is passed in and used only for migration
-		// otherwise, in stateless, we don't need the network id for deletion
-		epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID, args.Netns)
-		// if stateless CNI fail to get the endpoint from CNS for any reason other than Endpoint Not found or CNS connection failure
-		// return a retriable error so the container runtime will retry this DEL later
-		// the implementation of this function returns nil if the endpoint doesn't exist, so
-		// we don't have to check that here
-		if err != nil {
-			switch {
-			case errors.Is(err, network.ErrConnectionFailure):
-				logger.Error("Failed to connect to CNS", zap.Error(err))
-				logger.Info("Endpoint will be deleted from state file asynchronously", zap.String("containerID", args.ContainerID))
-				// In SwiftV2 Linux stateless CNI mode, if the plugin cannot connect to CNS,
-				// we asynchronously remove the secondary (delegated) interface from the pod’s network namespace in the absence of the endpoint state.
-				// This is necessary because leaving the delegated NIC in the pod netns can cause the kernel to block rtnetlink operations.
-				// When that happens, kubelet and containerd hang during sandbox creation or teardown.
-				// The delegated NIC (SR-IOV VF) used by SwiftV2 for multitenant pods remains tied to the pod namespace,
-				// triggering hot-unplug/re-register events and leaving the node in an unhealthy state.
-				// This workaround mitigates the issue by removing the secondary NIC from the pod netns when CNS is unreachable during DEL to provide the endpoint state.
-				if err = plugin.nm.RemoveSecondaryEndpointFromPodNetNS(args.IfName, args.Netns); err != nil {
-					logger.Error("Failed to remove secondary endpoint from pod netns", zap.String("netns", args.Netns), zap.Error(err))
-					return plugin.RetriableError(fmt.Errorf("failed to remove secondary endpoint from pod netns: %w", err))
-				}
-			case errors.Is(err, network.ErrEndpointStateNotFound):
-				logger.Info("Endpoint Not found", zap.String("containerID", args.ContainerID), zap.Error(err))
-				return nil
-			default:
-				logger.Error("Get Endpoint State API returned error", zap.String("containerID", args.ContainerID), zap.Error(err))
-				return plugin.RetriableError(fmt.Errorf("failed to delete endpoint: %w", err))
-			}
-		} else {
-			for _, epInfo := range epInfos {
-				logger.Info("Found endpoint to delete", zap.String("IfName", epInfo.IfName), zap.String("EndpointID", epInfo.EndpointID), zap.Any("NICType", epInfo.NICType))
-			}
-		}
-	} else {
-		epInfos = plugin.nm.GetEndpointInfosFromContainerID(args.ContainerID)
+	epInfos, err := plugin.nm.GetEndpointInfos(networkID, args, nwCfg.DisableAsyncDelete)
+	if err != nil {
+		return plugin.RetriableError(fmt.Errorf("failed to retrieve endpoint: %w", err))
 	}
-
-	// for Stateful CNI when the endpoint is not created, but the ips are already allocated (only works if single network, single infra)
-	// this block is applied to stateless CNI only if there was a connection failure in previous block and asynchronous delete by CNS will remover the endpoint from state file
+	// when the endpoint is not created, but the ips are already allocated (only works if single network, single infra)
 	if len(epInfos) == 0 {
 		endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName)
 		if !nwCfg.MultiTenancy {
@@ -1150,15 +1112,26 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
 			zap.String("endpointID", epInfo.EndpointID))
 		telemetryClient.SendEvent("Deleting endpoint: " + epInfo.EndpointID)
 
+		// Delegated/secondary nic ips are statically allocated so we don't need to release
+		// Call into IPAM plugin to release the endpoint's addresses.
 		if !nwCfg.MultiTenancy && (epInfo.NICType == cns.InfraNIC || epInfo.NICType == "") {
-			// Delegated/secondary nic ips are statically allocated so we don't need to release
-			// Call into IPAM plugin to release the endpoint's addresses.
-			for i := range epInfo.IPAddresses {
-				logger.Info("Release ip", zap.String("ip", epInfo.IPAddresses[i].IP.String()))
-				telemetryClient.SendEvent(fmt.Sprintf("Release ip: %s container id: %s endpoint id: %s", epInfo.IPAddresses[i].IP.String(), args.ContainerID, epInfo.EndpointID))
-				err = plugin.ipamInvoker.Delete(&epInfo.IPAddresses[i], nwCfg, args, nwInfo.Options)
-				if err != nil {
-					return plugin.RetriableError(fmt.Errorf("failed to release address: %w", err))
+			// This is an special case for stateless CNI when Asynchronous DEL to CNS will take place for SwiftV2 after the endpointinfo is recreated locally
+			// At this point the endpoint is already deleted and since it is created locally the IPAddress is nil. CNS will release the IP asynchronously whenever it is up
+			if epInfo.IPAddresses == nil && plugin.nm.IsStatelessCNIMode() && !nwCfg.DisableAsyncDelete {
+				logger.Warn("Release ip Asynchronously by CNS",
+					zap.String("containerID", args.ContainerID))
+				telemetryClient.SendEvent(fmt.Sprintf("Release ip: %s container id: %s endpoint id: %s asynchronously", epInfo.IPAddresses[i].IP.String(), args.ContainerID, epInfo.EndpointID))
+				if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil {
+					return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err))
+				}
+			} else {
+				for i := range epInfo.IPAddresses {
+					logger.Info("Release ip", zap.String("ip", epInfo.IPAddresses[i].IP.String()))
+					telemetryClient.SendEvent(fmt.Sprintf("Release ip: %s container id: %s endpoint id: %s", epInfo.IPAddresses[i].IP.String(), args.ContainerID, epInfo.EndpointID))
+					err = plugin.ipamInvoker.Delete(&epInfo.IPAddresses[i], nwCfg, args, nwInfo.Options)
+					if err != nil {
+						return plugin.RetriableError(fmt.Errorf("failed to release address: %w", err))
+					}
 				}
 			}
 		} else if epInfo.EnableInfraVnet { // remove in future PR

network/endpoint_linux.go

@@ -16,6 +16,7 @@ import (
 	"github.com/Azure/azure-container-networking/network/networkutils"
 	"github.com/Azure/azure-container-networking/ovsctl"
 	"github.com/Azure/azure-container-networking/platform"
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
 
@@ -548,11 +549,28 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(_ []net.IPNet, _ string) (*E
 	return epInfo, nil
 }
 
-// removeSecondaryEndpointFromPodNetNSImpl deletes an existing secondary endpoint from the pod network namespace.
-func (ep *endpoint) removeSecondaryEndpointFromPodNetNSImpl(nsc NamespaceClientInterface) error {
-	secondaryepClient := NewSecondaryEndpointClient(nil, nil, nil, nsc, nil, ep)
-	if err := secondaryepClient.RemoveInterfacesFromNetnsPath(ep.IfName, ep.NetworkNameSpace); err != nil {
-		return err
+// getEndpointInfoByIfNameImpl returns an array of EndpointInfo for the given endpoint based on the IfName(s) found in the network namespace.
+func (nm *networkManager) getEndpointInfoByIfNameImpl(ep *endpoint) ([]*EndpointInfo, error) {
+	epInfo := &EndpointInfo{
+		EndpointID: ep.Id,
+		NetNsPath:  ep.NetworkNameSpace,
+		NICType:    cns.InfraNIC,
+		IfName:     ep.IfName,
 	}
-	return nil
+	ret := []*EndpointInfo{}
+	ret = append(ret, epInfo)
+	logger.Info("Fetching Secondary Endpoint from", zap.String("NetworkNameSpace", ep.NetworkNameSpace))
+	secondaryepClient := NewSecondaryEndpointClient(nil, nil, nil, nm.nsClient, nil, ep)
+	ifnames, err := secondaryepClient.FetchInterfacesFromNetnsPath(ep.IfName, ep.NetworkNameSpace)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch secondary interfaces")
+	}
+	for _, ifName := range ifnames {
+		ret = append(ret, &EndpointInfo{
+			NetNsPath: ep.NetworkNameSpace,
+			IfName:    ifName,
+			NICType:   cns.NodeNetworkInterfaceFrontendNIC,
+		})
+	}
+	return ret, nil
 }

network/endpoint_windows.go

@@ -771,7 +771,6 @@ func getPnpDeviceState(instanceID string, plc platform.ExecClient) (string, stri
 	return devpkeyDeviceIsPresent, devpkeyDeviceProblemCode, nil
 }
 
-// removeSecondaryEndpointFromPodNetNSImpl removes an existing secondary endpoint from the pod network namespace.
-func (ep *endpoint) removeSecondaryEndpointFromPodNetNSImpl(_ NamespaceClientInterface) error {
-	return nil
+func (nm *networkManager) getEndpointInfoByIfNameImpl(_ *endpoint) ([]*EndpointInfo, error) {
+	return nil, nil
 }

network/errors.go

@@ -3,9 +3,11 @@ package network
 import "errors"
 
 var (
-	errSubnetV6NotFound        = errors.New("Couldn't find ipv6 subnet in network info")                // nolint
-	errV6SnatRuleNotSet        = errors.New("ipv6 snat rule not set. Might be VM ipv6 address missing") // nolint
-	ErrEndpointStateNotFound   = errors.New("endpoint state could not be found in the statefile")
-	ErrConnectionFailure       = errors.New("couldn't connect to CNS")
-	ErrGetEndpointStateFailure = errors.New("failure to obtain the endpoint state")
+	errSubnetV6NotFound         = errors.New("couldn't find ipv6 subnet in network info")                // nolint
+	errV6SnatRuleNotSet         = errors.New("ipv6 snat rule not set. Might be VM ipv6 address missing") // nolint
+	ErrEndpointStateNotFound    = errors.New("endpoint state could not be found in the statefile")
+	ErrConnectionFailure        = errors.New("couldn't connect to CNS")
+	ErrEndpointRemovalFailure   = errors.New("failed to remove endpoint")
+	ErrEndpointRetrievalFailure = errors.New("failed to obtain endpoint")
+	ErrGetEndpointStateFailure  = errors.New("failure to obtain the endpoint state")
 )

network/manager.go

@@ -19,6 +19,7 @@ import (
 	"github.com/Azure/azure-container-networking/netlink"
 	"github.com/Azure/azure-container-networking/platform"
 	"github.com/Azure/azure-container-networking/store"
+	cniSkel "github.com/containernetworking/cni/pkg/skel"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
@@ -120,9 +121,9 @@ type NetworkManager interface {
 	IsStatelessCNIMode() bool
 	SaveState(eps []*endpoint) error
 	DeleteState(epInfos []*EndpointInfo) error
+	GetEndpointInfos(networkID string, args *cniSkel.CmdArgs, disableAsyncDelete bool) ([]*EndpointInfo, error)
 	GetEndpointInfosFromContainerID(containerID string) []*EndpointInfo
 	GetEndpointState(networkID, containerID, netns string) ([]*EndpointInfo, error)
-	RemoveSecondaryEndpointFromPodNetNS(ifName string, netns string) error
 }
 
 // Creates a new network manager.
@@ -841,7 +842,9 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI
 		epInfo.HNSNetworkID = ipInfo.HnsNetworkID
 		epInfo.MacAddress = net.HardwareAddr(ipInfo.MacAddress)
 		epInfo.NetworkContainerID = ipInfo.NetworkContainerID
-
+		if epInfo.NetNsPath == "" {
+			epInfo.NetNsPath = netns
+		}
 		ret = append(ret, epInfo)
 	}
 	return ret
@@ -882,13 +885,57 @@ func generateCNSIPInfoMap(eps []*endpoint) map[string]*restserver.IPInfo {
 	return ifNametoIPInfoMap
 }
 
-// RemoveSecondaryEndpointFromPodNetNS removes the secondary endpoint from the pod netns
-func (nm *networkManager) RemoveSecondaryEndpointFromPodNetNS(ifName, netns string) error {
+// GetEndpointInfos gets all endpoint infos associated with a container id and networkID
+// In stateless CNI mode, it calls CNS GetEndpointState API to get the endpoint infos or genreate them locally if CNS is unreachable in SwiftV2 mode and AsyncDelete is enabled
+// In stateful CNI mode, it fetches the endpoint infos by calling GetEndpointInfosFromContainerID
+func (nm *networkManager) GetEndpointInfos(networkID string, args *cniSkel.CmdArgs, disableAsyncDelete bool) ([]*EndpointInfo, error) {
+	if nm.IsStatelessCNIMode() {
+		logger.Info("Calling cns getEndpoint API")
+		epInfos, err := nm.GetEndpointState(networkID, args.ContainerID, args.Netns)
+		if err != nil {
+			switch {
+			// async delete should be disabled for standalone scenarios but will be enabled for AKS scenarios
+			case errors.Is(err, ErrConnectionFailure) && !disableAsyncDelete:
+				logger.Info("Failed to connect to CNS, endpoint will be deleted from state file asynchronously", zap.String("containerID", args.ContainerID))
+				// In SwiftV2 Linux stateless CNI mode, if the plugin cannot connect to CNS,
+				// we still have to remove the secondary (delegated) interface from the pod’s network namespace in the absence of the endpoint state.
+				// This is necessary because leaving the delegated NIC in the pod netns can cause the kernel to block rtnetlink operations.
+				// When that happens, kubelet and containerd hang during sandbox creation or teardown.
+				// The delegated NIC (SR-IOV VF) used by SwiftV2 for multitenant pods remains tied to the pod namespace,
+				// triggering hot-unplug/re-register events and leaving the node in an unhealthy state.
+				// This workaround mitigates the issue by generating a minimal endpointInfo via containerd args and netlink APIs that can be then passed to DeleteEndpoint API.
+				epInfos, err = nm.generateEndpointLocally(args)
+				if err != nil {
+					logger.Error("Failed to fetch secondary endpoint from pod netns", zap.String("netns", args.Netns), zap.Error(err))
+					return nil, errors.Wrap(err, "failed to fetch secondary interfaces")
+				}
+			case errors.Is(err, ErrEndpointStateNotFound):
+				logger.Info("Endpoint Not found", zap.String("containerID", args.ContainerID), zap.Error(err))
+				return nil, nil
+			default:
+				logger.Error("Get Endpoint State API returned error", zap.String("containerID", args.ContainerID), zap.Error(err))
+				return nil, ErrEndpointRetrievalFailure
+			}
+		}
+		for _, epInfo := range epInfos {
+			logger.Info("Found endpoint to delete", zap.String("IfName", epInfo.IfName), zap.String("EndpointID", epInfo.EndpointID), zap.Any("NICType", epInfo.NICType))
+		}
+		return epInfos, nil
+	}
+	// Stateful CNI mode
+	return nm.GetEndpointInfosFromContainerID(args.ContainerID), nil
+}
+
+// generateEndpointLocally fetches the endpoint information using containerd args and netlink APIs
+func (nm *networkManager) generateEndpointLocally(args *cniSkel.CmdArgs) ([]*EndpointInfo, error) {
 	ep := &endpoint{
-		NetworkNameSpace: netns,
-		IfName:           ifName, // TODO: For stateless cni linux populate IfName here to use in deletion in secondary endpoint client
+		Id:               args.ContainerID,
+		NetworkNameSpace: args.Netns,
+		IfName:           args.IfName, // TODO: For stateless cni linux populate IfName here to use in deletion in secondary endpoint client
 	}
-	logger.Info("Removing Secondary Endpoint from", zap.String("NetworkNameSpace: ", netns))
-	err := ep.removeSecondaryEndpointFromPodNetNSImpl(nm.nsClient)
-	return err
+	epInfo, err := nm.getEndpointInfoByIfNameImpl(ep)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch secondary interfaces")
+	}
+	return epInfo, nil
 }

network/manager_mock.go

@@ -3,6 +3,7 @@ package network
 import (
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/common"
+	cniSkel "github.com/containernetworking/cni/pkg/skel"
 )
 
 // MockNetworkManager is a mock structure for Network Manager
@@ -222,6 +223,6 @@ func (nm *MockNetworkManager) GetEndpointState(_, _, _ string) ([]*EndpointInfo,
 	return []*EndpointInfo{}, nil
 }
 
-func (nm *MockNetworkManager) RemoveSecondaryEndpointFromPodNetNS(_, _ string) error {
-	return nil
+func (nm *MockNetworkManager) GetEndpointInfos(_ string, args *cniSkel.CmdArgs, _ bool) ([]*EndpointInfo, error) {
+	return nm.GetEndpointInfosFromContainerID(args.ContainerID), nil
 }