Skip to content

Commit 10c7e13

Browse files
arjan-balarjan-bal
authored
outlierdetection: Support health listener for ejection updates (#7908)
1 parent bce0535 commit 10c7e13

File tree

8 files changed

+487
-169
lines changed

8 files changed

+487
-169
lines changed

‎balancer/endpointsharding/endpointsharding.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,9 @@ import (
3535

3636
"google.golang.org/grpc/balancer"
3737
"google.golang.org/grpc/balancer/base"
38-
"google.golang.org/grpc/balancer/pickfirst"
3938
"google.golang.org/grpc/balancer/pickfirst/pickfirstleaf"
4039
"google.golang.org/grpc/connectivity"
4140
"google.golang.org/grpc/internal/balancer/gracefulswitch"
42-
"google.golang.org/grpc/internal/envconfig"
4341
"google.golang.org/grpc/resolver"
4442
"google.golang.org/grpc/serviceconfig"
4543
)
@@ -48,11 +46,7 @@ import (
4846
var PickFirstConfig string
4947

5048
func init() {
51-
name := pickfirst.Name
52-
if !envconfig.NewPickFirstEnabled {
53-
name = pickfirstleaf.Name
54-
}
55-
PickFirstConfig = fmt.Sprintf("[{%q: {}}]", name)
49+
PickFirstConfig = fmt.Sprintf("[{%q: {}}]", pickfirstleaf.Name)
5650
}
5751

5852
// ChildState is the balancer state of a child along with the endpoint which

‎balancer/pickfirst/pickfirstleaf/pickfirstleaf.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,18 @@ func init() {
5454
balancer.Register(pickfirstBuilder{})
5555
}
5656

57-
// enableHealthListenerKeyType is a unique key type used in resolver attributes
58-
// to indicate whether the health listener usage is enabled.
59-
type enableHealthListenerKeyType struct{}
57+
type (
58+
// enableHealthListenerKeyType is a unique key type used in resolver
59+
// attributes to indicate whether the health listener usage is enabled.
60+
enableHealthListenerKeyType struct{}
61+
// managedByPickfirstKeyType is an attribute key type to inform Outlier
62+
// Detection that the generic health listener is being used.
63+
// TODO: https://github.com/grpc/grpc-go/issues/7915 - Remove this when
64+
// implementing the dualstack design. This is a hack. Once Dualstack is
65+
// completed, outlier detection will stop sending ejection updates through
66+
// the connectivity listener.
67+
managedByPickfirstKeyType struct{}
68+
)
6069

6170
var (
6271
logger = grpclog.Component("pick-first-leaf-lb")
@@ -140,6 +149,17 @@ func EnableHealthListener(state resolver.State) resolver.State {
140149
return state
141150
}
142151

152+
// IsManagedByPickfirst returns whether an address belongs to a SubConn
153+
// managed by the pickfirst LB policy.
154+
// TODO: https://github.com/grpc/grpc-go/issues/7915 - This is a hack to disable
155+
// outlier_detection via the with connectivity listener when using pick_first.
156+
// Once Dualstack changes are complete, all SubConns will be created by
157+
// pick_first and outlier detection will only use the health listener for
158+
// ejection. This hack can then be removed.
159+
func IsManagedByPickfirst(addr resolver.Address) bool {
160+
return addr.BalancerAttributes.Value(managedByPickfirstKeyType{}) != nil
161+
}
162+
143163
type pfConfig struct {
144164
serviceconfig.LoadBalancingConfig `json:"-"`
145165

@@ -166,6 +186,7 @@ type scData struct {
166186
}
167187

168188
func (b *pickfirstBalancer) newSCData(addr resolver.Address) (*scData, error) {
189+
addr.BalancerAttributes = addr.BalancerAttributes.WithValue(managedByPickfirstKeyType{}, true)
169190
sd := &scData{
170191
rawConnectivityState: connectivity.Idle,
171192
effectiveState: connectivity.Idle,

‎balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,13 @@ const (
5858
stateStoringBalancerName = "state_storing"
5959
)
6060

61-
var stateStoringServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateStoringBalancerName)
61+
var (
62+
stateStoringServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateStoringBalancerName)
63+
ignoreBalAttributesOpt = cmp.Transformer("IgnoreBalancerAttributes", func(a resolver.Address) resolver.Address {
64+
a.BalancerAttributes = nil
65+
return a
66+
})
67+
)
6268

6369
type s struct {
6470
grpctest.Tester
@@ -177,7 +183,7 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerReady(t *testing.T) {
177183
wantSCStates := []scState{
178184
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready},
179185
}
180-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
186+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
181187
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
182188
}
183189

@@ -219,7 +225,7 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerUnReady(t *testing.T)
219225
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown},
220226
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
221227
}
222-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
228+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
223229
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
224230
}
225231

@@ -264,7 +270,7 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate_DuplicateAddrs(t *testing.T) {
264270
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown},
265271
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
266272
}
267-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
273+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
268274
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
269275
}
270276

@@ -317,7 +323,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) {
317323
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
318324
}
319325

320-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
326+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
321327
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
322328
}
323329

@@ -334,7 +340,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) {
334340
{Addrs: []resolver.Address{addrs[3]}, State: connectivity.Ready},
335341
}
336342

337-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
343+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
338344
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
339345
}
340346

@@ -378,7 +384,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing
378384
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
379385
}
380386

381-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
387+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
382388
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
383389
}
384390

@@ -398,7 +404,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing
398404
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
399405
}
400406

401-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
407+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
402408
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
403409
}
404410

@@ -440,7 +446,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi
440446
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
441447
}
442448

443-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
449+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
444450
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
445451
}
446452

@@ -458,7 +464,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi
458464
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready},
459465
}
460466

461-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
467+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
462468
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
463469
}
464470

@@ -502,7 +508,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) {
502508
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
503509
}
504510

505-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
511+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
506512
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
507513
}
508514

@@ -521,7 +527,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) {
521527
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
522528
}
523529

524-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
530+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
525531
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
526532
}
527533

@@ -576,7 +582,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T)
576582
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready},
577583
}
578584

579-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
585+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
580586
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
581587
}
582588

@@ -591,7 +597,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T)
591597
t.Fatal(err)
592598
}
593599

594-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
600+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
595601
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
596602
}
597603

@@ -639,7 +645,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T)
639645
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
640646
}
641647

642-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
648+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
643649
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
644650
}
645651

@@ -660,7 +666,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T)
660666
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown},
661667
}
662668

663-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
669+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
664670
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
665671
}
666672

@@ -708,7 +714,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T)
708714
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
709715
}
710716

711-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
717+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
712718
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
713719
}
714720

@@ -729,7 +735,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T)
729735
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready},
730736
}
731737

732-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
738+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
733739
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
734740
}
735741

@@ -776,7 +782,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T)
776782
{Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready},
777783
}
778784

779-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
785+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
780786
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
781787
}
782788

@@ -796,7 +802,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T)
796802
{Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready},
797803
}
798804

799-
if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" {
805+
if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" {
800806
t.Errorf("SubConn states mismatch (-want +got):\n%s", diff)
801807
}
802808

@@ -1130,7 +1136,7 @@ func (s) TestPickFirstLeaf_InterleavingIPV4Preffered(t *testing.T) {
11301136
if err != nil {
11311137
t.Fatalf("%v", err)
11321138
}
1133-
if diff := cmp.Diff(wantAddrs, gotAddrs); diff != "" {
1139+
if diff := cmp.Diff(wantAddrs, gotAddrs, ignoreBalAttributesOpt); diff != "" {
11341140
t.Errorf("SubConn creation order mismatch (-want +got):\n%s", diff)
11351141
}
11361142
}
@@ -1174,7 +1180,7 @@ func (s) TestPickFirstLeaf_InterleavingIPv6Preffered(t *testing.T) {
11741180
if err != nil {
11751181
t.Fatalf("%v", err)
11761182
}
1177-
if diff := cmp.Diff(wantAddrs, gotAddrs); diff != "" {
1183+
if diff := cmp.Diff(wantAddrs, gotAddrs, ignoreBalAttributesOpt); diff != "" {
11781184
t.Errorf("SubConn creation order mismatch (-want +got):\n%s", diff)
11791185
}
11801186
}
@@ -1220,7 +1226,7 @@ func (s) TestPickFirstLeaf_InterleavingUnknownPreffered(t *testing.T) {
12201226
if err != nil {
12211227
t.Fatalf("%v", err)
12221228
}
1223-
if diff := cmp.Diff(wantAddrs, gotAddrs); diff != "" {
1229+
if diff := cmp.Diff(wantAddrs, gotAddrs, ignoreBalAttributesOpt); diff != "" {
12241230
t.Errorf("SubConn creation order mismatch (-want +got):\n%s", diff)
12251231
}
12261232
}

‎balancer/weightedroundrobin/balancer.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
"google.golang.org/grpc/balancer"
3131
"google.golang.org/grpc/balancer/endpointsharding"
32+
"google.golang.org/grpc/balancer/pickfirst/pickfirstleaf"
3233
"google.golang.org/grpc/balancer/weightedroundrobin/internal"
3334
"google.golang.org/grpc/balancer/weightedtarget"
3435
"google.golang.org/grpc/connectivity"
@@ -218,7 +219,9 @@ type wrrBalancer struct {
218219
}
219220

220221
func (b *wrrBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error {
221-
b.logger.Infof("UpdateCCS: %v", ccs)
222+
if b.logger.V(2) {
223+
b.logger.Infof("UpdateCCS: %v", ccs)
224+
}
222225
cfg, ok := ccs.BalancerConfig.(*lbConfig)
223226
if !ok {
224227
return fmt.Errorf("wrr: received nil or illegal BalancerConfig (type %T): %v", ccs.BalancerConfig, ccs.BalancerConfig)
@@ -232,6 +235,9 @@ func (b *wrrBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error
232235
b.updateEndpointsLocked(ccs.ResolverState.Endpoints)
233236
b.mu.Unlock()
234237

238+
// Make pickfirst children use health listeners for outlier detection to
239+
// work.
240+
ccs.ResolverState = pickfirstleaf.EnableHealthListener(ccs.ResolverState)
235241
// This causes child to update picker inline and will thus cause inline
236242
// picker update.
237243
return b.child.UpdateClientConnState(balancer.ClientConnState{

0 commit comments

Comments
 (0)