Add envoy connection balancing. (#14616)

Add envoy connection balancing config.
hashicorp · Sep 26, 2022 · aa4709a · aa4709a
1 parent fc16f7a
commit aa4709a
Show file tree

Hide file tree

Showing 15 changed files with 528 additions and 83 deletions.
diff --git a/.changelog/14616.txt b/.changelog/14616.txt
@@ -0,0 +1,3 @@
+```release-note:feature
+connect: Add Envoy connection balancing configuration fields.
+```
diff --git a/agent/configentry/resolve.go b/agent/configentry/resolve.go
@@ -53,6 +53,7 @@ func ComputeResolvedServiceConfig(
 		structs.NewServiceID(args.Name, &args.EnterpriseMeta),
 	)
 	if serviceConf != nil {
+
 		if serviceConf.Expose.Checks {
 			thisReply.Expose.Checks = true
 		}
@@ -62,12 +63,6 @@ func ComputeResolvedServiceConfig(
 		if serviceConf.MeshGateway.Mode != structs.MeshGatewayModeDefault {
 			thisReply.MeshGateway.Mode = serviceConf.MeshGateway.Mode
 		}
-		if serviceConf.Protocol != "" {
-			if thisReply.ProxyConfig == nil {
-				thisReply.ProxyConfig = make(map[string]interface{})
-			}
-			thisReply.ProxyConfig["protocol"] = serviceConf.Protocol
-		}
 		if serviceConf.TransparentProxy.OutboundListenerPort != 0 {
 			thisReply.TransparentProxy.OutboundListenerPort = serviceConf.TransparentProxy.OutboundListenerPort
 		}
@@ -81,25 +76,29 @@ func ComputeResolvedServiceConfig(
 			thisReply.Destination = *serviceConf.Destination
 		}
 
+		// Populate values for the proxy config map
+		proxyConf := thisReply.ProxyConfig
+		if proxyConf == nil {
+			proxyConf = make(map[string]interface{})
+		}
+		if serviceConf.Protocol != "" {
+			proxyConf["protocol"] = serviceConf.Protocol
+		}
+		if serviceConf.BalanceInboundConnections != "" {
+			proxyConf["balance_inbound_connections"] = serviceConf.BalanceInboundConnections
+		}
 		if serviceConf.MaxInboundConnections > 0 {
-			if thisReply.ProxyConfig == nil {
-				thisReply.ProxyConfig = map[string]interface{}{}
-			}
-			thisReply.ProxyConfig["max_inbound_connections"] = serviceConf.MaxInboundConnections
+			proxyConf["max_inbound_connections"] = serviceConf.MaxInboundConnections
 		}
-
 		if serviceConf.LocalConnectTimeoutMs > 0 {
-			if thisReply.ProxyConfig == nil {
-				thisReply.ProxyConfig = map[string]interface{}{}
-			}
-			thisReply.ProxyConfig["local_connect_timeout_ms"] = serviceConf.LocalConnectTimeoutMs
+			proxyConf["local_connect_timeout_ms"] = serviceConf.LocalConnectTimeoutMs
 		}
-
 		if serviceConf.LocalRequestTimeoutMs > 0 {
-			if thisReply.ProxyConfig == nil {
-				thisReply.ProxyConfig = map[string]interface{}{}
-			}
-			thisReply.ProxyConfig["local_request_timeout_ms"] = serviceConf.LocalRequestTimeoutMs
+			proxyConf["local_request_timeout_ms"] = serviceConf.LocalRequestTimeoutMs
+		}
+		// Add the proxy conf to the response if any fields were populated
+		if len(proxyConf) > 0 {
+			thisReply.ProxyConfig = proxyConf
 		}
 
 		thisReply.Meta = serviceConf.Meta

diff --git a/agent/configentry/resolve_test.go b/agent/configentry/resolve_test.go
@@ -24,6 +24,26 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) {
 		args args
 		want *structs.ServiceConfigResponse
 	}{
+		{
+			name: "proxy with balanceinboundconnections",
+			args: args{
+				scReq: &structs.ServiceConfigRequest{
+					Name: "sid",
+				},
+				entries: &ResolvedServiceConfigSet{
+					ServiceDefaults: map[structs.ServiceID]*structs.ServiceConfigEntry{
+						sid: {
+							BalanceInboundConnections: "exact_balance",
+						},
+					},
+				},
+			},
+			want: &structs.ServiceConfigResponse{
+				ProxyConfig: map[string]interface{}{
+					"balance_inbound_connections": "exact_balance",
+				},
+			},
+		},
 		{
 			name: "proxy with maxinboundsconnections",
 			args: args{

diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go
@@ -38,6 +38,8 @@ const (
 	MeshConfigMesh    string = "mesh"
 
 	DefaultServiceProtocol = "tcp"
+
+	ConnectionExactBalance = "exact_balance"
 )
 
 var AllConfigEntryKinds = []string{
@@ -98,19 +100,20 @@ type WarningConfigEntry interface {
 // ServiceConfiguration is the top-level struct for the configuration of a service
 // across the entire cluster.
 type ServiceConfigEntry struct {
-	Kind                  string
-	Name                  string
-	Protocol              string
-	Mode                  ProxyMode              `json:",omitempty"`
-	TransparentProxy      TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
-	MeshGateway           MeshGatewayConfig      `json:",omitempty" alias:"mesh_gateway"`
-	Expose                ExposeConfig           `json:",omitempty"`
-	ExternalSNI           string                 `json:",omitempty" alias:"external_sni"`
-	UpstreamConfig        *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"`
-	Destination           *DestinationConfig     `json:",omitempty"`
-	MaxInboundConnections int                    `json:",omitempty" alias:"max_inbound_connections"`
-	LocalConnectTimeoutMs int                    `json:",omitempty" alias:"local_connect_timeout_ms"`
-	LocalRequestTimeoutMs int                    `json:",omitempty" alias:"local_request_timeout_ms"`
+	Kind                      string
+	Name                      string
+	Protocol                  string
+	Mode                      ProxyMode              `json:",omitempty"`
+	TransparentProxy          TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
+	MeshGateway               MeshGatewayConfig      `json:",omitempty" alias:"mesh_gateway"`
+	Expose                    ExposeConfig           `json:",omitempty"`
+	ExternalSNI               string                 `json:",omitempty" alias:"external_sni"`
+	UpstreamConfig            *UpstreamConfiguration `json:",omitempty" alias:"upstream_config"`
+	Destination               *DestinationConfig     `json:",omitempty"`
+	MaxInboundConnections     int                    `json:",omitempty" alias:"max_inbound_connections"`
+	LocalConnectTimeoutMs     int                    `json:",omitempty" alias:"local_connect_timeout_ms"`
+	LocalRequestTimeoutMs     int                    `json:",omitempty" alias:"local_request_timeout_ms"`
+	BalanceInboundConnections string                 `json:",omitempty" alias:"balance_inbound_connections"`
 
 	Meta               map[string]string `json:",omitempty"`
 	acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
@@ -183,6 +186,10 @@ func (e *ServiceConfigEntry) Validate() error {
 
 	validationErr := validateConfigEntryMeta(e.Meta)
 
+	if !isValidConnectionBalance(e.BalanceInboundConnections) {
+		validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_inbound_connections: %v", e.BalanceInboundConnections))
+	}
+
 	// External endpoints are invalid with an existing service's upstream configuration
 	if e.UpstreamConfig != nil && e.Destination != nil {
 		validationErr = multierror.Append(validationErr, errors.New("UpstreamConfig and Destination are mutually exclusive for service defaults"))
@@ -800,6 +807,10 @@ type UpstreamConfig struct {
 
 	// MeshGatewayConfig controls how Mesh Gateways are configured and used
 	MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway" `
+
+	// BalanceOutboundConnections indicates how the proxy should attempt to distribute
+	// connections across worker threads. Only used by envoy proxies.
+	BalanceOutboundConnections string `json:",omitempty" alias:"balance_outbound_connections"`
 }
 
 func (cfg UpstreamConfig) Clone() UpstreamConfig {
@@ -848,6 +859,9 @@ func (cfg UpstreamConfig) MergeInto(dst map[string]interface{}) {
 	if cfg.PassiveHealthCheck != nil {
 		dst["passive_health_check"] = cfg.PassiveHealthCheck
 	}
+	if cfg.BalanceOutboundConnections != "" {
+		dst["balance_outbound_connections"] = cfg.BalanceOutboundConnections
+	}
 }
 
 func (cfg *UpstreamConfig) NormalizeWithoutName() error {
@@ -917,6 +931,10 @@ func (cfg UpstreamConfig) validate(named bool) error {
 		}
 	}
 
+	if !isValidConnectionBalance(cfg.BalanceOutboundConnections) {
+		validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_outbound_connections: %v", cfg.BalanceOutboundConnections))
+	}
+
 	return validationErr
 }
 
@@ -1222,3 +1240,7 @@ func validateConfigEntryMeta(meta map[string]string) error {
 type ConfigEntryDeleteResponse struct {
 	Deleted bool
 }
+
+func isValidConnectionBalance(s string) bool {
+	return s == "" || s == ConnectionExactBalance
+}