Skip to content

Commit aca07ce

Browse files
zasweqzasweq
authored
xds/internal/xdsclient: Add least request support in xDS (#6517)
1 parent e5d8eac commit aca07ce

File tree

9 files changed

+188
-12
lines changed

9 files changed

+188
-12
lines changed

‎balancer/leastrequest/leastrequest.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import (
3131
"google.golang.org/grpc/serviceconfig"
3232
)
3333

34-
// Global to stub out in tests.
34+
// grpcranduint32 is a global to stub out in tests.
3535
var grpcranduint32 = grpcrand.Uint32
3636

3737
// Name is the name of the least request balancer.

‎internal/envconfig/envconfig.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ var (
3939
// PickFirstLBConfig is set if we should support configuration of the
4040
// pick_first LB policy.
4141
PickFirstLBConfig = boolFromEnv("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", true)
42+
// LeastRequestLB is set if we should support the least_request_experimental
43+
// LB policy, which can be enabled by setting the environment variable
44+
// "GRPC_EXPERIMENTAL_ENABLE_LEAST_REQUEST" to "true".
45+
LeastRequestLB = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_LEAST_REQUEST", false)
4246
// ALTSMaxConcurrentHandshakes is the maximum number of concurrent ALTS
4347
// handshakes that can be performed.
4448
ALTSMaxConcurrentHandshakes = uint64FromEnv("GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES", 100, 1, 100)

‎test/xds/xds_client_custom_lb_test.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"time"
2626

2727
"google.golang.org/grpc"
28+
_ "google.golang.org/grpc/balancer/leastrequest" // To register least_request
2829
_ "google.golang.org/grpc/balancer/weightedroundrobin" // To register weighted_round_robin
2930
"google.golang.org/grpc/credentials/insecure"
3031
"google.golang.org/grpc/internal/envconfig"
@@ -41,6 +42,7 @@ import (
4142
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
4243
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
4344
v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
45+
v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3"
4446
v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3"
4547
v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
4648
"github.com/golang/protobuf/proto"
@@ -96,6 +98,11 @@ func (s) TestWrrLocality(t *testing.T) {
9698
defer func() {
9799
envconfig.XDSCustomLBPolicy = oldCustomLBSupport
98100
}()
101+
oldLeastRequestLBSupport := envconfig.LeastRequestLB
102+
envconfig.LeastRequestLB = true
103+
defer func() {
104+
envconfig.LeastRequestLB = oldLeastRequestLBSupport
105+
}()
99106

100107
backend1 := stubserver.StartTestService(t, nil)
101108
port1 := testutils.ParsePort(t, backend1.Address)
@@ -194,12 +201,33 @@ func (s) TestWrrLocality(t *testing.T) {
194201
{addr: backend5.Address, count: 8},
195202
},
196203
},
204+
{
205+
name: "custom_lb_least_request",
206+
wrrLocalityConfiguration: wrrLocality(&v3leastrequestpb.LeastRequest{
207+
ChoiceCount: wrapperspb.UInt32(2),
208+
}),
209+
// The test performs a Unary RPC, and blocks until the RPC returns,
210+
// and then makes the next Unary RPC. Thus, over iterations, no RPC
211+
// counts are present. This causes least request's randomness of
212+
// indexes to sample to converge onto a round robin distribution per
213+
// locality. Thus, expect the same distribution as round robin
214+
// above.
215+
addressDistributionWant: []struct {
216+
addr string
217+
count int
218+
}{
219+
{addr: backend1.Address, count: 6},
220+
{addr: backend2.Address, count: 6},
221+
{addr: backend3.Address, count: 8},
222+
{addr: backend4.Address, count: 8},
223+
{addr: backend5.Address, count: 8},
224+
},
225+
},
197226
}
198227
for _, test := range tests {
199228
t.Run(test.name, func(t *testing.T) {
200229
managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{})
201230
defer cleanup()
202-
203231
routeConfigName := "route-" + serviceName
204232
clusterName := "cluster-" + serviceName
205233
endpointsName := "endpoints-" + serviceName

‎xds/internal/balancer/balancer.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package balancer
2121

2222
import (
23+
_ "google.golang.org/grpc/balancer/leastrequest" // Register the least_request_experimental balancer
2324
_ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer
2425
_ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // Register the CDS balancer
2526
_ "google.golang.org/grpc/xds/internal/balancer/clusterimpl" // Register the xds_cluster_impl balancer

‎xds/internal/xdsclient/xdslbregistry/converter/converter.go

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/golang/protobuf/proto"
3131
"google.golang.org/grpc"
3232
"google.golang.org/grpc/balancer"
33+
"google.golang.org/grpc/balancer/leastrequest"
3334
"google.golang.org/grpc/balancer/roundrobin"
3435
"google.golang.org/grpc/balancer/weightedroundrobin"
3536
"google.golang.org/grpc/internal/envconfig"
@@ -41,6 +42,7 @@ import (
4142
v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1"
4243
v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
4344
v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
45+
v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3"
4446
v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3"
4547
v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3"
4648
v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
@@ -53,13 +55,15 @@ func init() {
5355
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.pick_first.v3.PickFirst", convertPickFirstProtoToServiceConfig)
5456
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin", convertRoundRobinProtoToServiceConfig)
5557
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.wrr_locality.v3.WrrLocality", convertWRRLocalityProtoToServiceConfig)
58+
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest", convertLeastRequestProtoToServiceConfig)
5659
xdslbregistry.Register("type.googleapis.com/udpa.type.v1.TypedStruct", convertV1TypedStructToServiceConfig)
5760
xdslbregistry.Register("type.googleapis.com/xds.type.v3.TypedStruct", convertV3TypedStructToServiceConfig)
5861
}
5962

6063
const (
61-
defaultRingHashMinSize = 1024
62-
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M
64+
defaultRingHashMinSize = 1024
65+
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M
66+
defaultLeastRequestChoiceCount = 2
6367
)
6468

6569
func convertRingHashProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
@@ -177,6 +181,29 @@ func convertWeightedRoundRobinProtoToServiceConfig(rawProto []byte, _ int) (json
177181
return makeBalancerConfigJSON(weightedroundrobin.Name, lbCfgJSON), nil
178182
}
179183

184+
func convertLeastRequestProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
185+
if !envconfig.LeastRequestLB {
186+
return nil, nil
187+
}
188+
lrProto := &v3leastrequestpb.LeastRequest{}
189+
if err := proto.Unmarshal(rawProto, lrProto); err != nil {
190+
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
191+
}
192+
// "The configuration for the Least Request LB policy is the
193+
// least_request_lb_config field. The field is optional; if not present,
194+
// defaults will be assumed for all of its values." - A48
195+
choiceCount := uint32(defaultLeastRequestChoiceCount)
196+
if cc := lrProto.GetChoiceCount(); cc != nil {
197+
choiceCount = cc.GetValue()
198+
}
199+
lrCfg := &leastrequest.LBConfig{ChoiceCount: choiceCount}
200+
js, err := json.Marshal(lrCfg)
201+
if err != nil {
202+
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", lrCfg, err)
203+
}
204+
return makeBalancerConfigJSON(leastrequest.Name, js), nil
205+
}
206+
180207
func convertV1TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
181208
tsProto := &v1xdsudpatypepb.TypedStruct{}
182209
if err := proto.Unmarshal(rawProto, tsProto); err != nil {

‎xds/internal/xdsclient/xdslbregistry/xdslbregistry_test.go

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ func wrrLocalityBalancerConfig(childPolicy *internalserviceconfig.BalancerConfig
6969
}
7070

7171
func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
72+
defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB)
73+
envconfig.LeastRequestLB = false
74+
7275
const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy"
7376
stub.Register(customLBPolicyName, stub.BalancerFuncs{})
7477

@@ -78,6 +81,7 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
7881
wantConfig string // JSON config
7982
rhDisabled bool
8083
pfDisabled bool
84+
lrEnabled bool
8185
}{
8286
{
8387
name: "ring_hash",
@@ -96,6 +100,22 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
96100
},
97101
wantConfig: `[{"ring_hash_experimental": { "minRingSize": 10, "maxRingSize": 100 }}]`,
98102
},
103+
{
104+
name: "least_request",
105+
policy: &v3clusterpb.LoadBalancingPolicy{
106+
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
107+
{
108+
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
109+
TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{
110+
ChoiceCount: wrapperspb.UInt32(3),
111+
}),
112+
},
113+
},
114+
},
115+
},
116+
wantConfig: `[{"least_request_experimental": { "choiceCount": 3 }}]`,
117+
lrEnabled: true,
118+
},
99119
{
100120
name: "pick_first_shuffle",
101121
policy: &v3clusterpb.LoadBalancingPolicy{
@@ -183,7 +203,7 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
183203
rhDisabled: true,
184204
},
185205
{
186-
name: "pick_first_disabled_pf_rr_use_first_supported",
206+
name: "pick_first_enabled_pf_rr_use_pick_first",
187207
policy: &v3clusterpb.LoadBalancingPolicy{
188208
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
189209
{
@@ -200,17 +220,16 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
200220
},
201221
},
202222
},
203-
wantConfig: `[{"round_robin": {}}]`,
204-
pfDisabled: true,
223+
wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`,
205224
},
206225
{
207-
name: "pick_first_enabled_pf_rr_use_pick_first",
226+
name: "least_request_disabled_pf_rr_use_first_supported",
208227
policy: &v3clusterpb.LoadBalancingPolicy{
209228
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
210229
{
211230
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
212-
TypedConfig: testutils.MarshalAny(&v3pickfirstpb.PickFirst{
213-
ShuffleAddressList: true,
231+
TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{
232+
ChoiceCount: wrapperspb.UInt32(32),
214233
}),
215234
},
216235
},
@@ -221,7 +240,7 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
221240
},
222241
},
223242
},
224-
wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`,
243+
wantConfig: `[{"round_robin": {}}]`,
225244
},
226245
{
227246
name: "custom_lb_type_v3_struct",
@@ -317,6 +336,10 @@ func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
317336
defer func(old bool) { envconfig.XDSRingHash = old }(envconfig.XDSRingHash)
318337
envconfig.XDSRingHash = false
319338
}
339+
if test.lrEnabled {
340+
defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB)
341+
envconfig.LeastRequestLB = true
342+
}
320343
if test.pfDisabled {
321344
defer func(old bool) { envconfig.PickFirstLBConfig = old }(envconfig.PickFirstLBConfig)
322345
envconfig.PickFirstLBConfig = false

‎xds/internal/xdsclient/xdsresource/tests/unmarshal_cds_test.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"github.com/google/go-cmp/cmp"
2727
"github.com/google/go-cmp/cmp/cmpopts"
28+
"google.golang.org/grpc/balancer/leastrequest"
2829
_ "google.golang.org/grpc/balancer/roundrobin" // To register round_robin load balancer.
2930
"google.golang.org/grpc/internal/balancer/stub"
3031
"google.golang.org/grpc/internal/envconfig"
@@ -103,6 +104,8 @@ func (s) TestValidateCluster_Success(t *testing.T) {
103104
defer func() {
104105
envconfig.XDSCustomLBPolicy = origCustomLBSupport
105106
}()
107+
defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB)
108+
envconfig.LeastRequestLB = true
106109
tests := []struct {
107110
name string
108111
cluster *v3clusterpb.Cluster
@@ -330,6 +333,31 @@ func (s) TestValidateCluster_Success(t *testing.T) {
330333
},
331334
},
332335
},
336+
{
337+
name: "happiest-case-with-least-request-lb-policy-with-default-config",
338+
cluster: &v3clusterpb.Cluster{
339+
Name: clusterName,
340+
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
341+
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
342+
EdsConfig: &v3corepb.ConfigSource{
343+
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
344+
Ads: &v3corepb.AggregatedConfigSource{},
345+
},
346+
},
347+
ServiceName: serviceName,
348+
},
349+
LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
350+
},
351+
wantUpdate: xdsresource.ClusterUpdate{
352+
ClusterName: clusterName, EDSServiceName: serviceName,
353+
},
354+
wantLBConfig: &iserviceconfig.BalancerConfig{
355+
Name: "least_request_experimental",
356+
Config: &leastrequest.LBConfig{
357+
ChoiceCount: 2,
358+
},
359+
},
360+
},
333361
{
334362
name: "happiest-case-with-ring-hash-lb-policy-with-none-default-config",
335363
cluster: &v3clusterpb.Cluster{
@@ -367,6 +395,36 @@ func (s) TestValidateCluster_Success(t *testing.T) {
367395
},
368396
},
369397
},
398+
{
399+
name: "happiest-case-with-least-request-lb-policy-with-none-default-config",
400+
cluster: &v3clusterpb.Cluster{
401+
Name: clusterName,
402+
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
403+
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
404+
EdsConfig: &v3corepb.ConfigSource{
405+
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
406+
Ads: &v3corepb.AggregatedConfigSource{},
407+
},
408+
},
409+
ServiceName: serviceName,
410+
},
411+
LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
412+
LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{
413+
LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{
414+
ChoiceCount: wrapperspb.UInt32(3),
415+
},
416+
},
417+
},
418+
wantUpdate: xdsresource.ClusterUpdate{
419+
ClusterName: clusterName, EDSServiceName: serviceName,
420+
},
421+
wantLBConfig: &iserviceconfig.BalancerConfig{
422+
Name: "least_request_experimental",
423+
Config: &leastrequest.LBConfig{
424+
ChoiceCount: 3,
425+
},
426+
},
427+
},
370428
{
371429
name: "happiest-case-with-ring-hash-lb-policy-configured-through-LoadBalancingPolicy",
372430
cluster: &v3clusterpb.Cluster{

‎xds/internal/xdsclient/xdsresource/unmarshal_cds.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ const (
7676
defaultRingHashMinSize = 1024
7777
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M
7878
ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M
79+
80+
defaultLeastRequestChoiceCount = 2
7981
)
8082

8183
func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) {
@@ -104,6 +106,26 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu
104106

105107
rhLBCfg := []byte(fmt.Sprintf("{\"minRingSize\": %d, \"maxRingSize\": %d}", minSize, maxSize))
106108
lbPolicy = []byte(fmt.Sprintf(`[{"ring_hash_experimental": %s}]`, rhLBCfg))
109+
case v3clusterpb.Cluster_LEAST_REQUEST:
110+
if !envconfig.LeastRequestLB {
111+
return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
112+
}
113+
114+
// "The configuration for the Least Request LB policy is the
115+
// least_request_lb_config field. The field is optional; if not present,
116+
// defaults will be assumed for all of its values." - A48
117+
lr := cluster.GetLeastRequestLbConfig()
118+
var choiceCount uint32 = defaultLeastRequestChoiceCount
119+
if cc := lr.GetChoiceCount(); cc != nil {
120+
choiceCount = cc.GetValue()
121+
}
122+
// "If choice_count < 2, the config will be rejected." - A48
123+
if choiceCount < 2 {
124+
return ClusterUpdate{}, fmt.Errorf("Cluster_LeastRequestLbConfig.ChoiceCount must be >= 2, got: %v", choiceCount)
125+
}
126+
127+
lrLBCfg := []byte(fmt.Sprintf("{\"choiceCount\": %d}", choiceCount))
128+
lbPolicy = []byte(fmt.Sprintf(`[{"least_request_experimental": %s}]`, lrLBCfg))
107129
default:
108130
return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
109131
}

‎xds/internal/xdsclient/xdsresource/unmarshal_cds_test.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,19 @@ func (s) TestValidateCluster_Failure(t *testing.T) {
161161
wantUpdate: emptyUpdate,
162162
wantErr: true,
163163
},
164+
{
165+
name: "least-request-choice-count-less-than-two",
166+
cluster: &v3clusterpb.Cluster{
167+
LbPolicy: v3clusterpb.Cluster_RING_HASH,
168+
LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{
169+
LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{
170+
ChoiceCount: wrapperspb.UInt32(1),
171+
},
172+
},
173+
},
174+
wantUpdate: emptyUpdate,
175+
wantErr: true,
176+
},
164177
{
165178
name: "ring-hash-max-bound-greater-than-upper-bound",
166179
cluster: &v3clusterpb.Cluster{
@@ -205,7 +218,7 @@ func (s) TestValidateCluster_Failure(t *testing.T) {
205218
wantErr: true,
206219
},
207220
{
208-
name: "least-request-unsupported-in-converter",
221+
name: "least-request-unsupported-in-converter-since-env-var-unset",
209222
cluster: &v3clusterpb.Cluster{
210223
Name: clusterName,
211224
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},

0 commit comments

Comments
 (0)