Add extra source addresses for Inbound cluster via upstream bindconfi…

…g to support dual stack in Istio (istio#41618)

* Add extra source addresses for Inbound cluster via upstream bindconfig to support dual stack in Istio

* add release notes for dual stack support

* fixed issues based on comments

* fixed lint-go

* remove TODO for reachability test

* add feature flags EnableDualStack to support Dual Stack for both istiod and istio-agent, merge inboundPassthroughCluster into one once Dual Stack is enabled.

* fixed the dual stack integration test

* remove prefix ForDualStack for functions: getWildcardsAndLocalHost and getPassthroughBindIPs

* change code based on comments

* change the inbound passthrough cluster name and let getLocalIP return another for ipv6 enable.

* add go comments for how to implement the smarter pick between SourceAddress and ExtraSourceAddresses for Dual Stack support via bindconfig

* Update releasenotes/notes/40394.yaml

highlight the feature flags

Co-authored-by: Greg Hanson <gregory.hanson@solo.io>

Co-authored-by: Greg Hanson <gregory.hanson@solo.io>

pilot/pkg/features/pilot.go

@@ -689,6 +689,9 @@ var (
 		false,
 		"If set, it allows creating inbound listeners for service ports and sidecar ingress listeners ",
 	).Get()
+
+	EnableDualStack = env.RegisterBoolVar("ISTIO_DUAL_STACK", false,
+		"If enabled, pilot will configure clusters/listeners/routes for dual stack capability.").Get()
 )
 
 // EnableEndpointSliceController returns the value of the feature flag and whether it was actually specified.

pilot/pkg/networking/core/v1alpha3/cluster_builder.go

@@ -99,22 +99,22 @@ type metadataCerts struct {
 // ClusterBuilder interface provides an abstraction for building Envoy Clusters.
 type ClusterBuilder struct {
 	// Proxy related information used to build clusters.
-	serviceInstances  []*model.ServiceInstance // Service instances of Proxy.
-	metadataCerts     *metadataCerts           // Client certificates specified in metadata.
-	clusterID         string                   // Cluster in which proxy is running.
-	proxyID           string                   // Identifier that uniquely identifies a proxy.
-	proxyVersion      string                   // Version of Proxy.
-	proxyType         model.NodeType           // Indicates whether the proxy is sidecar or gateway.
-	sidecarScope      *model.SidecarScope      // Computed sidecar for the proxy.
-	passThroughBindIP string                   // Passthrough IP to be used while building clusters.
-	supportsIPv4      bool                     // Whether Proxy IPs has IPv4 address.
-	supportsIPv6      bool                     // Whether Proxy IPs has IPv6 address.
-	hbone             bool                     // Does the proxy support HBONE
-	locality          *core.Locality           // Locality information of proxy.
-	proxyLabels       map[string]string        // Proxy labels.
-	proxyView         model.ProxyView          // Proxy view of endpoints.
-	proxyIPAddresses  []string                 // IP addresses on which proxy is listening on.
-	configNamespace   string                   // Proxy config namespace.
+	serviceInstances   []*model.ServiceInstance // Service instances of Proxy.
+	metadataCerts      *metadataCerts           // Client certificates specified in metadata.
+	clusterID          string                   // Cluster in which proxy is running.
+	proxyID            string                   // Identifier that uniquely identifies a proxy.
+	proxyVersion       string                   // Version of Proxy.
+	proxyType          model.NodeType           // Indicates whether the proxy is sidecar or gateway.
+	sidecarScope       *model.SidecarScope      // Computed sidecar for the proxy.
+	passThroughBindIPs []string                 // Passthrough IPs to be used while building clusters.
+	supportsIPv4       bool                     // Whether Proxy IPs has IPv4 address.
+	supportsIPv6       bool                     // Whether Proxy IPs has IPv6 address.
+	hbone              bool                     // Does the proxy support HBONE
+	locality           *core.Locality           // Locality information of proxy.
+	proxyLabels        map[string]string        // Proxy labels.
+	proxyView          model.ProxyView          // Proxy view of endpoints.
+	proxyIPAddresses   []string                 // IP addresses on which proxy is listening on.
+	configNamespace    string                   // Proxy config namespace.
 	// PushRequest to look for updates.
 	req                   *model.PushRequest
 	cache                 model.XdsCache
@@ -124,23 +124,23 @@ type ClusterBuilder struct {
 // NewClusterBuilder builds an instance of ClusterBuilder.
 func NewClusterBuilder(proxy *model.Proxy, req *model.PushRequest, cache model.XdsCache) *ClusterBuilder {
 	cb := &ClusterBuilder{
-		serviceInstances:  proxy.ServiceInstances,
-		proxyID:           proxy.ID,
-		proxyType:         proxy.Type,
-		proxyVersion:      proxy.Metadata.IstioVersion,
-		sidecarScope:      proxy.SidecarScope,
-		passThroughBindIP: getPassthroughBindIP(proxy),
-		supportsIPv4:      proxy.SupportsIPv4(),
-		supportsIPv6:      proxy.SupportsIPv6(),
-		hbone:             proxy.EnableHBONE(),
-		locality:          proxy.Locality,
-		proxyLabels:       proxy.Labels,
-		proxyView:         proxy.GetView(),
-		proxyIPAddresses:  proxy.IPAddresses,
-		configNamespace:   proxy.ConfigNamespace,
-		req:               req,
-		cache:             cache,
-	}
+		serviceInstances: proxy.ServiceInstances,
+		proxyID:          proxy.ID,
+		proxyType:        proxy.Type,
+		proxyVersion:     proxy.Metadata.IstioVersion,
+		sidecarScope:     proxy.SidecarScope,
+		supportsIPv4:     proxy.SupportsIPv4(),
+		supportsIPv6:     proxy.SupportsIPv6(),
+		hbone:            proxy.EnableHBONE(),
+		locality:         proxy.Locality,
+		proxyLabels:      proxy.Labels,
+		proxyView:        proxy.GetView(),
+		proxyIPAddresses: proxy.IPAddresses,
+		configNamespace:  proxy.ConfigNamespace,
+		req:              req,
+		cache:            cache,
+	}
+	cb.passThroughBindIPs = getPassthroughBindIPs(proxy.GetIPMode())
 	if proxy.Metadata != nil {
 		if proxy.Metadata.TLSClientCertChain != "" {
 			cb.metadataCerts = &metadataCerts{
@@ -471,12 +471,26 @@ func (cb *ClusterBuilder) buildInboundClusterForPortOrUDS(clusterPort int, bind
 		// config which will be skipped.
 		localCluster.cluster.UpstreamBindConfig = &core.BindConfig{
 			SourceAddress: &core.SocketAddress{
-				Address: cb.passThroughBindIP,
+				Address: cb.passThroughBindIPs[0],
 				PortSpecifier: &core.SocketAddress_PortValue{
 					PortValue: uint32(0),
 				},
 			},
 		}
+		// add extra source addresses to cluster builder
+		var extraSrcAddrs []*core.ExtraSourceAddress
+		for _, extraBdIP := range cb.passThroughBindIPs[1:] {
+			extraSrcAddr := &core.ExtraSourceAddress{
+				Address: &core.SocketAddress{
+					Address: extraBdIP,
+					PortSpecifier: &core.SocketAddress_PortValue{
+						PortValue: uint32(0),
+					},
+				},
+			}
+			extraSrcAddrs = append(extraSrcAddrs, extraSrcAddr)
+		}
+		localCluster.cluster.UpstreamBindConfig.ExtraSourceAddresses = extraSrcAddrs
 	}
 	return localCluster
 }
@@ -592,33 +606,63 @@ func addUint32(left, right uint32) (uint32, bool) {
 func (cb *ClusterBuilder) buildInboundPassthroughClusters() []*cluster.Cluster {
 	// ipv4 and ipv6 feature detection. Envoy cannot ignore a config where the ip version is not supported
 	clusters := make([]*cluster.Cluster, 0, 2)
-	if cb.supportsIPv4 {
-		inboundPassthroughClusterIpv4 := cb.buildDefaultPassthroughCluster()
-		inboundPassthroughClusterIpv4.Name = util.InboundPassthroughClusterIpv4
-		inboundPassthroughClusterIpv4.Filters = nil
-		inboundPassthroughClusterIpv4.UpstreamBindConfig = &core.BindConfig{
+	// There is an usage doc here:
+	// https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#config-core-v3-bindconfig
+	// to support the Dual Stack via Envoy bindconfig, and belows are related issue and PR in Envoy:
+	// https://github.com/envoyproxy/envoy/issues/9811
+	// https://github.com/envoyproxy/envoy/pull/22639
+	if features.EnableDualStack {
+		inboundPassthroughClusterForDS := cb.buildDefaultPassthroughCluster()
+		inboundPassthroughClusterForDS.Name = util.InboundPassthroughClusterDualStack
+		inboundPassthroughClusterForDS.Filters = nil
+		inboundPassthroughClusterForDS.UpstreamBindConfig = &core.BindConfig{
 			SourceAddress: &core.SocketAddress{
 				Address: InboundPassthroughBindIpv4,
 				PortSpecifier: &core.SocketAddress_PortValue{
 					PortValue: uint32(0),
 				},
 			},
-		}
-		clusters = append(clusters, inboundPassthroughClusterIpv4)
-	}
-	if cb.supportsIPv6 {
-		inboundPassthroughClusterIpv6 := cb.buildDefaultPassthroughCluster()
-		inboundPassthroughClusterIpv6.Name = util.InboundPassthroughClusterIpv6
-		inboundPassthroughClusterIpv6.Filters = nil
-		inboundPassthroughClusterIpv6.UpstreamBindConfig = &core.BindConfig{
-			SourceAddress: &core.SocketAddress{
-				Address: InboundPassthroughBindIpv6,
-				PortSpecifier: &core.SocketAddress_PortValue{
-					PortValue: uint32(0),
+			ExtraSourceAddresses: []*core.ExtraSourceAddress{
+				{
+					Address: &core.SocketAddress{
+						Address: InboundPassthroughBindIpv6,
+						PortSpecifier: &core.SocketAddress_PortValue{
+							PortValue: uint32(0),
+						},
+					},
 				},
 			},
 		}
-		clusters = append(clusters, inboundPassthroughClusterIpv6)
+		clusters = append(clusters, inboundPassthroughClusterForDS)
+	} else {
+		if cb.supportsIPv4 {
+			inboundPassthroughClusterIpv4 := cb.buildDefaultPassthroughCluster()
+			inboundPassthroughClusterIpv4.Name = util.InboundPassthroughClusterIpv4
+			inboundPassthroughClusterIpv4.Filters = nil
+			inboundPassthroughClusterIpv4.UpstreamBindConfig = &core.BindConfig{
+				SourceAddress: &core.SocketAddress{
+					Address: InboundPassthroughBindIpv4,
+					PortSpecifier: &core.SocketAddress_PortValue{
+						PortValue: uint32(0),
+					},
+				},
+			}
+			clusters = append(clusters, inboundPassthroughClusterIpv4)
+		}
+		if cb.supportsIPv6 {
+			inboundPassthroughClusterIpv6 := cb.buildDefaultPassthroughCluster()
+			inboundPassthroughClusterIpv6.Name = util.InboundPassthroughClusterIpv6
+			inboundPassthroughClusterIpv6.Filters = nil
+			inboundPassthroughClusterIpv6.UpstreamBindConfig = &core.BindConfig{
+				SourceAddress: &core.SocketAddress{
+					Address: InboundPassthroughBindIpv6,
+					PortSpecifier: &core.SocketAddress_PortValue{
+						PortValue: uint32(0),
+					},
+				},
+			}
+			clusters = append(clusters, inboundPassthroughClusterIpv6)
+		}
 	}
 	return clusters
 }

pilot/pkg/networking/core/v1alpha3/listener.go

@@ -270,7 +270,7 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
 ) []*listener.Listener {
 	noneMode := node.GetInterceptionMode() == model.InterceptionNone
 
-	actualWildcards, actualLocalHosts := getWildcardsAndLocalHostForDualStack(node.GetIPMode())
+	actualWildcards, actualLocalHosts := getWildcardsAndLocalHost(node.GetIPMode())
 
 	var tcpListeners, httpListeners []*listener.Listener
 	// For conflict resolution
@@ -529,7 +529,7 @@ func (lb *ListenerBuilder) buildHTTPProxy(node *model.Proxy,
 	}
 
 	// enable HTTP PROXY port if necessary; this will add an RDS route for this port
-	_, actualLocalHosts := getWildcardsAndLocalHostForDualStack(node.GetIPMode())
+	_, actualLocalHosts := getWildcardsAndLocalHost(node.GetIPMode())
 
 	httpOpts := &core.Http1ProtocolOptions{
 		AllowAbsoluteUrl: proto.BoolTrue,
@@ -585,7 +585,7 @@ func buildSidecarOutboundHTTPListenerOptsForPortOrUDS(listenerMapKey *string,
 	// first identify the bind if its not set. Then construct the key
 	// used to lookup the listener in the conflict map.
 	if len(listenerOpts.bind) == 0 { // no user specified bind. Use 0.0.0.0:Port or [::]:Port
-		actualWildcards, _ := getWildcardsAndLocalHostForDualStack(listenerOpts.proxy.GetIPMode())
+		actualWildcards, _ := getWildcardsAndLocalHost(listenerOpts.proxy.GetIPMode())
 		listenerOpts.bind = actualWildcards[0]
 		if len(actualWildcards) > 0 {
 			listenerOpts.extraBind = actualWildcards[1:]
@@ -1238,7 +1238,9 @@ func buildListener(opts buildListenerOpts, trafficDirection core.TrafficDirectio
 			ConnectionBalanceConfig: connectionBalance,
 		}
 		// add extra addresses for the listener
-		res.AdditionalAddresses = util.BuildAdditionalAddresses(opts.extraBind, uint32(opts.port.Port), opts.proxy)
+		if features.EnableDualStack && len(opts.extraBind) > 0 {
+			res.AdditionalAddresses = util.BuildAdditionalAddresses(opts.extraBind, uint32(opts.port.Port), opts.proxy)
+		}
 
 		if opts.proxy.Type != model.Router {
 			res.ListenerFiltersTimeout = opts.push.Mesh.ProtocolDetectionTimeout
@@ -1267,7 +1269,9 @@ func buildListener(opts buildListenerOpts, trafficDirection core.TrafficDirectio
 			EnableReusePort: proto.BoolTrue,
 		}
 		// add extra addresses for the listener
-		res.AdditionalAddresses = util.BuildAdditionalAddresses(opts.extraBind, uint32(opts.port.Port), opts.proxy)
+		if features.EnableDualStack && len(opts.extraBind) > 0 {
+			res.AdditionalAddresses = util.BuildAdditionalAddresses(opts.extraBind, uint32(opts.port.Port), opts.proxy)
+		}
 	}
 
 	accessLogBuilder.setListenerAccessLog(opts.push, opts.proxy, res, opts.class)

pilot/pkg/networking/core/v1alpha3/listener_address.go

@@ -15,6 +15,7 @@
 package v1alpha3
 
 import (
+	"istio.io/istio/pilot/pkg/features"
 	"istio.io/istio/pilot/pkg/model"
 )
 
@@ -37,8 +38,24 @@ const (
 )
 
 var (
-	wildCards  map[model.IPMode][]string
-	localHosts map[model.IPMode][]string
+	// maintain 3 maps to return wildCards, localHosts and passthroughBindIPs according to IP mode of proxy
+	wildCards = map[model.IPMode][]string{
+		model.IPv4: {WildcardAddress},
+		model.IPv6: {WildcardIPv6Address},
+		model.Dual: {WildcardAddress, WildcardIPv6Address},
+	}
+
+	localHosts = map[model.IPMode][]string{
+		model.IPv4: {LocalhostAddress},
+		model.IPv6: {LocalhostIPv6Address},
+		model.Dual: {LocalhostAddress, LocalhostIPv6Address},
+	}
+
+	passthroughBindIPs = map[model.IPMode][]string{
+		model.IPv4: {InboundPassthroughBindIpv4},
+		model.IPv6: {InboundPassthroughBindIpv6},
+		model.Dual: {InboundPassthroughBindIpv4, InboundPassthroughBindIpv6},
+	}
 )
 
 // TODO: getActualWildcardAndLocalHost would be removed once the dual stack support in Istio
@@ -51,11 +68,19 @@ func getActualWildcardAndLocalHost(node *model.Proxy) (string, string) {
 	return WildcardIPv6Address, LocalhostIPv6Address
 }
 
-func getPassthroughBindIP(node *model.Proxy) string {
-	if node.SupportsIPv4() {
-		return InboundPassthroughBindIpv4
+func getPassthroughBindIPs(ipMode model.IPMode) []string {
+	passthroughBindIPAddresses := passthroughBindIPs[ipMode]
+
+	// it means that ipMode is empty if passthroughBindIPAddresses is empty
+	if len(passthroughBindIPAddresses) == 0 {
+		return []string{InboundPassthroughBindIpv4}
 	}
-	return InboundPassthroughBindIpv6
+
+	if !features.EnableDualStack && ipMode == model.Dual {
+		return []string{InboundPassthroughBindIpv4}
+	}
+
+	return passthroughBindIPAddresses
 }
 
 // getSidecarInboundBindIPs returns the IP that the proxy can bind to along with the sidecar specified port.
@@ -66,24 +91,10 @@ func getSidecarInboundBindIPs(node *model.Proxy) []string {
 	if len(node.GlobalUnicastIP) > 0 {
 		return []string{node.GlobalUnicastIP}
 	}
-	defaultInboundIPs, _ := getWildcardsAndLocalHostForDualStack(node.GetIPMode())
+	defaultInboundIPs, _ := getWildcardsAndLocalHost(node.GetIPMode())
 	return defaultInboundIPs
 }
 
-func getWildcardsAndLocalHostForDualStack(ipMode model.IPMode) ([]string, []string) {
+func getWildcardsAndLocalHost(ipMode model.IPMode) ([]string, []string) {
 	return wildCards[ipMode], localHosts[ipMode]
 }
-
-func init() {
-	// maintain 2 maps to return wildCards and localHosts according to IP mode of proxy
-	wildCards = make(map[model.IPMode][]string)
-	localHosts = make(map[model.IPMode][]string)
-
-	wildCards[model.IPv4] = []string{WildcardAddress}
-	wildCards[model.IPv6] = []string{WildcardIPv6Address}
-	wildCards[model.Dual] = []string{WildcardAddress, WildcardIPv6Address}
-
-	localHosts[model.IPv4] = []string{LocalhostAddress}
-	localHosts[model.IPv6] = []string{LocalhostIPv6Address}
-	localHosts[model.Dual] = []string{LocalhostAddress, LocalhostIPv6Address}
-}

pilot/pkg/networking/core/v1alpha3/listener_builder.go

@@ -127,7 +127,7 @@ func (lb *ListenerBuilder) buildVirtualOutboundListener() *ListenerBuilder {
 
 	filterChains := buildOutboundCatchAllNetworkFilterChains(lb.node, lb.push)
 
-	actualWildcards, _ := getWildcardsAndLocalHostForDualStack(lb.node.GetIPMode())
+	actualWildcards, _ := getWildcardsAndLocalHost(lb.node.GetIPMode())
 	// add an extra listener that binds to the port that is the recipient of the iptables redirect
 	ipTablesListener := &listener.Listener{
 		Name:             model.VirtualOutboundListenerName,
@@ -138,7 +138,7 @@ func (lb *ListenerBuilder) buildVirtualOutboundListener() *ListenerBuilder {
 		TrafficDirection: core.TrafficDirection_OUTBOUND,
 	}
 	// add extra addresses for the listener
-	if len(actualWildcards) > 1 {
+	if features.EnableDualStack && len(actualWildcards) > 1 {
 		ipTablesListener.AdditionalAddresses = util.BuildAdditionalAddresses(actualWildcards[1:], uint32(lb.push.Mesh.ProxyListenPort), lb.node)
 	}
 

pilot/pkg/networking/core/v1alpha3/listener_inbound.go

@@ -305,7 +305,7 @@ func (lb *ListenerBuilder) buildInboundListeners() []*listener.Listener {
 
 // inboundVirtualListener builds the virtual inbound listener.
 func (lb *ListenerBuilder) inboundVirtualListener(chains []*listener.FilterChain) *listener.Listener {
-	actualWildcards, _ := getWildcardsAndLocalHostForDualStack(lb.node.GetIPMode())
+	actualWildcards, _ := getWildcardsAndLocalHost(lb.node.GetIPMode())
 
 	// Build the "virtual" inbound listener. This will capture all inbound redirected traffic and contains:
 	// * Passthrough filter chains, matching all unmatched traffic. There are a few of these to handle all cases
@@ -338,7 +338,7 @@ func (lb *ListenerBuilder) buildInboundListener(name string, addresses []string,
 		Address:          address,
 		TrafficDirection: core.TrafficDirection_INBOUND,
 	}
-	if len(addresses) > 1 {
+	if features.EnableDualStack && len(addresses) > 1 {
 		// add extra addresses for the listener
 		l.AdditionalAddresses = util.BuildAdditionalAddresses(addresses[1:], tPort, lb.node)
 	}
@@ -410,7 +410,7 @@ func (lb *ListenerBuilder) getFilterChainsByServicePort(chainsByPort map[uint32]
 			TargetPort: i.Endpoint.EndpointPort,
 			Protocol:   i.ServicePort.Protocol,
 		}
-		actualWildcards, _ := getWildcardsAndLocalHostForDualStack(lb.node.GetIPMode())
+		actualWildcards, _ := getWildcardsAndLocalHost(lb.node.GetIPMode())
 		if enableSidecarServiceInboundListenerMerge && sidecarScope.HasIngressListener() &&
 			ingressPortListSet.Contains(int(port.Port)) {
 			// here if port is declared in service and sidecar ingress both, we continue to take the one on sidecar + other service ports

pilot/pkg/networking/core/v1alpha3/listener_test.go

@@ -976,7 +976,7 @@ func TestGetDualStackActualWildcard(t *testing.T) {
 	}
 	for _, tt := range tests {
 		tt.proxy.DiscoverIPMode()
-		actualWildcards, _ := getWildcardsAndLocalHostForDualStack(tt.proxy.GetIPMode())
+		actualWildcards, _ := getWildcardsAndLocalHost(tt.proxy.GetIPMode())
 		if len(actualWildcards) != len(tt.expected) {
 			t.Errorf("Test %s failed, expected: %v got: %v", tt.name, tt.expected, actualWildcards)
 		}
@@ -1013,7 +1013,7 @@ func TestGetDualStackLocalHost(t *testing.T) {
 	}
 	for _, tt := range tests {
 		tt.proxy.DiscoverIPMode()
-		_, actualLocalHosts := getWildcardsAndLocalHostForDualStack(tt.proxy.GetIPMode())
+		_, actualLocalHosts := getWildcardsAndLocalHost(tt.proxy.GetIPMode())
 		if len(actualLocalHosts) != len(tt.expected) {
 			t.Errorf("Test %s failed, expected: %v got: %v", tt.name, tt.expected, actualLocalHosts)
 		}