From c9864a11988193f317826e6bc052dfa070b7b5dd Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Tue, 8 Aug 2023 13:21:03 -0700 Subject: [PATCH 01/14] android-interop-testing: Remove usage of Netty for grpc server We have the OkHttp server these days, so we don't need to use Netty. Use the generic API instead of hard-coding OkHttp. We've seen some recent interop failures. We aren't entirely sure what is going on, but we have seen some Netty usages in logcat. Since we don't even want Netty on Android, just get rid of it and even if it doesn't help with the failures things are better dependency-wise. --- android-interop-testing/build.gradle | 3 +-- .../grpc/android/integrationtest/UdsChannelInteropTest.java | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/android-interop-testing/build.gradle b/android-interop-testing/build.gradle index feccf6040bb..69f5a62bfaa 100644 --- a/android-interop-testing/build.gradle +++ b/android-interop-testing/build.gradle @@ -88,8 +88,7 @@ dependencies { compileOnly libraries.javax.annotation - androidTestImplementation project(':grpc-netty'), - 'androidx.test.ext:junit:1.1.3', + androidTestImplementation 'androidx.test.ext:junit:1.1.3', 'androidx.test:runner:1.4.0' } diff --git a/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java b/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java index f002dd291c7..f5e54da5d4e 100644 --- a/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java +++ b/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java @@ -22,9 +22,10 @@ import androidx.test.InstrumentationRegistry; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.rule.ActivityTestRule; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; import io.grpc.Server; import io.grpc.android.UdsChannelBuilder; -import io.grpc.netty.NettyServerBuilder; import io.grpc.testing.integration.TestServiceImpl; import java.io.IOException; import java.util.concurrent.ExecutionException; @@ -68,7 +69,7 @@ public void setUp() throws IOException { // Start local server. server = - NettyServerBuilder.forPort(0) + Grpc.newServerBuilderForPort(0, InsecureServerCredentials.create()) .maxInboundMessageSize(16 * 1024 * 1024) .addService(new TestServiceImpl(serverExecutor)) .build(); From 4453ce7eb6748cfdb4fa1c5de9b9ca95ebe11536 Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Tue, 8 Aug 2023 14:35:44 -0700 Subject: [PATCH 02/14] util: Outlier detection tracer delegation (#10459) OutlierDetectionLoadBalancer did not delegate calls to an existing ClientStreamTracer from the tracer it installed. This change has the OD tracer delegate all calls to the underlying one. --- .../util/OutlierDetectionLoadBalancer.java | 58 ++++++++------ .../OutlierDetectionLoadBalancerTest.java | 80 ++++++++++++++++++- 2 files changed, 111 insertions(+), 27 deletions(-) diff --git a/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java b/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java index a1a4b4be282..bd8825474fa 100644 --- a/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java +++ b/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java @@ -394,47 +394,55 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { Subchannel subchannel = pickResult.getSubchannel(); if (subchannel != null) { - return PickResult.withSubchannel(subchannel, - new ResultCountingClientStreamTracerFactory( - subchannel.getAttributes().get(ADDRESS_TRACKER_ATTR_KEY))); + return PickResult.withSubchannel(subchannel, new ResultCountingClientStreamTracerFactory( + subchannel.getAttributes().get(ADDRESS_TRACKER_ATTR_KEY), + pickResult.getStreamTracerFactory())); } return pickResult; } /** - * Builds instances of {@link ResultCountingClientStreamTracer}. + * Builds instances of a {@link ClientStreamTracer} that increments the call count in the + * tracker for each closed stream. */ class ResultCountingClientStreamTracerFactory extends ClientStreamTracer.Factory { private final AddressTracker tracker; - ResultCountingClientStreamTracerFactory(AddressTracker tracker) { - this.tracker = tracker; - } - - @Override - public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) { - return new ResultCountingClientStreamTracer(tracker); - } - } + @Nullable + private final ClientStreamTracer.Factory delegateFactory; - /** - * Counts the results (successful/unsuccessful) of a particular {@link - * OutlierDetectionSubchannel}s streams and increments the counter in the associated {@link - * AddressTracker}. - */ - class ResultCountingClientStreamTracer extends ClientStreamTracer { - - AddressTracker tracker; - - public ResultCountingClientStreamTracer(AddressTracker tracker) { + ResultCountingClientStreamTracerFactory(AddressTracker tracker, + @Nullable ClientStreamTracer.Factory delegateFactory) { this.tracker = tracker; + this.delegateFactory = delegateFactory; } @Override - public void streamClosed(Status status) { - tracker.incrementCallCount(status.isOk()); + public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) { + if (delegateFactory != null) { + ClientStreamTracer delegateTracer = delegateFactory.newClientStreamTracer(info, headers); + return new ForwardingClientStreamTracer() { + @Override + protected ClientStreamTracer delegate() { + return delegateTracer; + } + + @Override + public void streamClosed(Status status) { + tracker.incrementCallCount(status.isOk()); + delegate().streamClosed(status); + } + }; + } else { + return new ClientStreamTracer() { + @Override + public void streamClosed(Status status) { + tracker.incrementCallCount(status.isOk()); + } + }; + } } } } diff --git a/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java b/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java index 18f9bbf549f..13f13421a1e 100644 --- a/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java +++ b/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; @@ -46,6 +47,7 @@ import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancer.SubchannelStateListener; import io.grpc.LoadBalancerProvider; +import io.grpc.Metadata; import io.grpc.Status; import io.grpc.SynchronizationContext; import io.grpc.internal.FakeClock; @@ -96,6 +98,10 @@ public class OutlierDetectionLoadBalancerTest { private Helper mockHelper; @Mock private SocketAddress mockSocketAddress; + @Mock + private ClientStreamTracer.Factory mockStreamTracerFactory; + @Mock + private ClientStreamTracer mockStreamTracer; @Captor private ArgumentCaptor connectivityStateCaptor; @@ -193,6 +199,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable { } }); + when(mockStreamTracerFactory.newClientStreamTracer(any(), + any())).thenReturn(mockStreamTracer); + loadBalancer = new OutlierDetectionLoadBalancer(mockHelper, fakeClock.getTimeProvider()); } @@ -355,6 +364,72 @@ public void delegatePick() throws Exception { readySubchannel); } + /** + * Any ClientStreamTracer.Factory set by the delegate picker should still get used. + */ + @Test + public void delegatePickTracerFactoryPreserved() throws Exception { + OutlierDetectionLoadBalancerConfig config = new OutlierDetectionLoadBalancerConfig.Builder() + .setSuccessRateEjection(new SuccessRateEjection.Builder().build()) + .setChildPolicy(new PolicySelection(fakeLbProvider, null)).build(); + + loadBalancer.acceptResolvedAddresses(buildResolvedAddress(config, servers.get(0))); + + // Make one of the subchannels READY. + final Subchannel readySubchannel = subchannels.values().iterator().next(); + deliverSubchannelState(readySubchannel, ConnectivityStateInfo.forNonError(READY)); + + verify(mockHelper, times(2)).updateBalancingState(stateCaptor.capture(), + pickerCaptor.capture()); + + // Make sure that we can pick the single READY subchannel. + SubchannelPicker picker = pickerCaptor.getAllValues().get(1); + PickResult pickResult = picker.pickSubchannel(mock(PickSubchannelArgs.class)); + + // Calls to a stream tracer created with the factory in the result should make it to a stream + // tracer the underlying LB/picker is using. + ClientStreamTracer clientStreamTracer = pickResult.getStreamTracerFactory() + .newClientStreamTracer(ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + clientStreamTracer.inboundHeaders(); + // The underlying fake LB provider is configured with a factory that returns a mock stream + // tracer. + verify(mockStreamTracer).inboundHeaders(); + } + + /** + * Assure the tracer works even when the underlying LB does not have a tracer to delegate to. + */ + @Test + public void delegatePickTracerFactoryNotSet() throws Exception { + // We set the mock factory to null to indicate that the delegate does not have its own tracer. + mockStreamTracerFactory = null; + + OutlierDetectionLoadBalancerConfig config = new OutlierDetectionLoadBalancerConfig.Builder() + .setSuccessRateEjection(new SuccessRateEjection.Builder().build()) + .setChildPolicy(new PolicySelection(fakeLbProvider, null)).build(); + + loadBalancer.acceptResolvedAddresses(buildResolvedAddress(config, servers.get(0))); + + // Make one of the subchannels READY. + final Subchannel readySubchannel = subchannels.values().iterator().next(); + deliverSubchannelState(readySubchannel, ConnectivityStateInfo.forNonError(READY)); + + verify(mockHelper, times(2)).updateBalancingState(stateCaptor.capture(), + pickerCaptor.capture()); + + // Make sure that we can pick the single READY subchannel. + SubchannelPicker picker = pickerCaptor.getAllValues().get(1); + PickResult pickResult = picker.pickSubchannel(mock(PickSubchannelArgs.class)); + + // With no delegate tracers factory a call to the OD tracer should still work + ClientStreamTracer clientStreamTracer = pickResult.getStreamTracerFactory() + .newClientStreamTracer(ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + clientStreamTracer.inboundHeaders(); + + // Sanity check to make sure the delegate tracer does not get called. + verifyNoInteractions(mockStreamTracer); + } + /** * The success rate algorithm leaves a healthy set of addresses alone. */ @@ -1121,7 +1196,7 @@ void assertEjectedSubchannels(Set addresses) { } /** Round robin like fake load balancer. */ - private static final class FakeLoadBalancer extends LoadBalancer { + private final class FakeLoadBalancer extends LoadBalancer { private final Helper helper; List subchannelList; @@ -1159,7 +1234,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { if (lastPickIndex < 0 || lastPickIndex > subchannelList.size() - 1) { lastPickIndex = 0; } - return PickResult.withSubchannel(subchannelList.get(lastPickIndex++)); + return PickResult.withSubchannel(subchannelList.get(lastPickIndex++), + mockStreamTracerFactory); } }; helper.updateBalancingState(state, picker); From a0d8f2eb31c88cf8e9e8d2f213a8cc8ccfb814b0 Mon Sep 17 00:00:00 2001 From: Larry Safran Date: Tue, 8 Aug 2023 21:36:27 +0000 Subject: [PATCH 03/14] Create a shared LB base class for LBs with multiple children (#10448) * Create a shared LB base class for LBs with multiple children and change ClusgterManagerLoadBalancer to use it. --- api/src/main/java/io/grpc/LoadBalancer.java | 35 +++ .../io/grpc/util/MultiChildLoadBalancer.java | 264 ++++++++++++++++++ .../java/io/grpc/xds/CdsLoadBalancer2.java | 1 - .../io/grpc/xds/ClusterImplLoadBalancer.java | 4 +- .../grpc/xds/ClusterManagerLoadBalancer.java | 247 ++-------------- .../grpc/xds/ClusterResolverLoadBalancer.java | 1 - .../io/grpc/xds/PriorityLoadBalancer.java | 6 +- .../io/grpc/xds/RingHashLoadBalancer.java | 1 - .../grpc/xds/WeightedTargetLoadBalancer.java | 6 +- .../io/grpc/xds/WrrLocalityLoadBalancer.java | 1 - .../io/grpc/xds/XdsSubchannelPickers.java | 63 ----- .../xds/ClusterManagerLoadBalancerTest.java | 39 ++- .../io/grpc/xds/PriorityLoadBalancerTest.java | 26 +- .../xds/WeightedTargetLoadBalancerTest.java | 16 +- .../grpc/xds/WrrLocalityLoadBalancerTest.java | 2 +- 15 files changed, 373 insertions(+), 339 deletions(-) create mode 100644 util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java delete mode 100644 xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java diff --git a/api/src/main/java/io/grpc/LoadBalancer.java b/api/src/main/java/io/grpc/LoadBalancer.java index 5617d279862..d7e3fbb917c 100644 --- a/api/src/main/java/io/grpc/LoadBalancer.java +++ b/api/src/main/java/io/grpc/LoadBalancer.java @@ -115,6 +115,19 @@ public abstract class LoadBalancer { @NameResolver.ResolutionResultAttr public static final Attributes.Key> ATTR_HEALTH_CHECKING_CONFIG = Attributes.Key.create("internal:health-checking-config"); + + public static final SubchannelPicker EMPTY_PICKER = new SubchannelPicker() { + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withNoResult(); + } + + @Override + public String toString() { + return "EMPTY_PICKER"; + } + }; + private int recursionCount; /** @@ -1398,4 +1411,26 @@ public abstract static class Factory { */ public abstract LoadBalancer newLoadBalancer(Helper helper); } + + public static final class ErrorPicker extends SubchannelPicker { + + private final Status error; + + public ErrorPicker(Status error) { + this.error = checkNotNull(error, "error"); + } + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withError(error); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("error", error) + .toString(); + } + } + } diff --git a/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java b/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java new file mode 100644 index 00000000000..3671505a345 --- /dev/null +++ b/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java @@ -0,0 +1,264 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.util; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.IDLE; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; + +import com.google.common.annotations.VisibleForTesting; +import io.grpc.ConnectivityState; +import io.grpc.LoadBalancer; +import io.grpc.LoadBalancerProvider; +import io.grpc.Status; +import io.grpc.SynchronizationContext; +import io.grpc.SynchronizationContext.ScheduledHandle; +import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * A base load balancing policy for those policies which has multiple children such as + * ClusterManager or the petiole policies. + * + * @since 1.58 + */ +public abstract class MultiChildLoadBalancer extends LoadBalancer { + + @VisibleForTesting + public static final int DELAYED_CHILD_DELETION_TIME_MINUTES = 15; + private static final Logger logger = Logger.getLogger(MultiChildLoadBalancer.class.getName()); + private final Map childLbStates = new HashMap<>(); + private final Helper helper; + protected final SynchronizationContext syncContext; + private final ScheduledExecutorService timeService; + // Set to true if currently in the process of handling resolved addresses. + private boolean resolvingAddresses; + + protected MultiChildLoadBalancer(Helper helper) { + this.helper = checkNotNull(helper, "helper"); + this.syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext"); + this.timeService = checkNotNull(helper.getScheduledExecutorService(), "timeService"); + logger.log(Level.FINE, "Created"); + } + + protected SubchannelPicker getInitialPicker() { + return EMPTY_PICKER; + } + + protected SubchannelPicker getErrorPicker(Status error) { + return new ErrorPicker(error); + } + + protected abstract Map getPolicySelectionMap( + ResolvedAddresses resolvedAddresses); + + protected abstract SubchannelPicker getSubchannelPicker( + Map childPickers); + + @Override + public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { + try { + resolvingAddresses = true; + return acceptResolvedAddressesInternal(resolvedAddresses); + } finally { + resolvingAddresses = false; + } + } + + private boolean acceptResolvedAddressesInternal(ResolvedAddresses resolvedAddresses) { + logger.log(Level.FINE, "Received resolution result: {0}", resolvedAddresses); + Map newChildPolicies = getPolicySelectionMap(resolvedAddresses); + for (Map.Entry entry : newChildPolicies.entrySet()) { + final Object key = entry.getKey(); + LoadBalancerProvider childPolicyProvider = entry.getValue().getProvider(); + Object childConfig = entry.getValue().getConfig(); + if (!childLbStates.containsKey(key)) { + childLbStates.put(key, new ChildLbState(key, childPolicyProvider, getInitialPicker())); + } else { + childLbStates.get(key).reactivate(childPolicyProvider); + } + LoadBalancer childLb = childLbStates.get(key).lb; + ResolvedAddresses childAddresses = + resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(childConfig).build(); + childLb.handleResolvedAddresses(childAddresses); + } + for (Object key : childLbStates.keySet()) { + if (!newChildPolicies.containsKey(key)) { + childLbStates.get(key).deactivate(); + } + } + // Must update channel picker before return so that new RPCs will not be routed to deleted + // clusters and resolver can remove them in service config. + updateOverallBalancingState(); + return true; + } + + @Override + public void handleNameResolutionError(Status error) { + logger.log(Level.WARNING, "Received name resolution error: {0}", error); + boolean gotoTransientFailure = true; + for (ChildLbState state : childLbStates.values()) { + if (!state.deactivated) { + gotoTransientFailure = false; + state.lb.handleNameResolutionError(error); + } + } + if (gotoTransientFailure) { + helper.updateBalancingState(TRANSIENT_FAILURE, getErrorPicker(error)); + } + } + + @Override + public void shutdown() { + logger.log(Level.INFO, "Shutdown"); + for (ChildLbState state : childLbStates.values()) { + state.shutdown(); + } + childLbStates.clear(); + } + + private void updateOverallBalancingState() { + ConnectivityState overallState = null; + final Map childPickers = new HashMap<>(); + for (ChildLbState childLbState : childLbStates.values()) { + if (childLbState.deactivated) { + continue; + } + childPickers.put(childLbState.key, childLbState.currentPicker); + overallState = aggregateState(overallState, childLbState.currentState); + } + if (overallState != null) { + helper.updateBalancingState(overallState, getSubchannelPicker(childPickers)); + } + } + + @Nullable + private static ConnectivityState aggregateState( + @Nullable ConnectivityState overallState, ConnectivityState childState) { + if (overallState == null) { + return childState; + } + if (overallState == READY || childState == READY) { + return READY; + } + if (overallState == CONNECTING || childState == CONNECTING) { + return CONNECTING; + } + if (overallState == IDLE || childState == IDLE) { + return IDLE; + } + return overallState; + } + + private final class ChildLbState { + private final Object key; + private final GracefulSwitchLoadBalancer lb; + private LoadBalancerProvider policyProvider; + private ConnectivityState currentState = CONNECTING; + private SubchannelPicker currentPicker; + private boolean deactivated; + @Nullable + ScheduledHandle deletionTimer; + + ChildLbState(Object key, LoadBalancerProvider policyProvider, SubchannelPicker initialPicker) { + this.key = key; + this.policyProvider = policyProvider; + lb = new GracefulSwitchLoadBalancer(new ChildLbStateHelper()); + lb.switchTo(policyProvider); + currentPicker = initialPicker; + } + + void deactivate() { + if (deactivated) { + return; + } + + class DeletionTask implements Runnable { + @Override + public void run() { + shutdown(); + childLbStates.remove(key); + } + } + + deletionTimer = + syncContext.schedule( + new DeletionTask(), + DELAYED_CHILD_DELETION_TIME_MINUTES, + TimeUnit.MINUTES, + timeService); + deactivated = true; + logger.log(Level.FINE, "Child balancer {0} deactivated", key); + } + + void reactivate(LoadBalancerProvider policyProvider) { + if (deletionTimer != null && deletionTimer.isPending()) { + deletionTimer.cancel(); + deactivated = false; + logger.log(Level.FINE, "Child balancer {0} reactivated", key); + } + if (!this.policyProvider.getPolicyName().equals(policyProvider.getPolicyName())) { + Object[] objects = { + key, this.policyProvider.getPolicyName(),policyProvider.getPolicyName()}; + logger.log(Level.FINE, "Child balancer {0} switching policy from {1} to {2}", objects); + lb.switchTo(policyProvider); + this.policyProvider = policyProvider; + } + } + + void shutdown() { + if (deletionTimer != null && deletionTimer.isPending()) { + deletionTimer.cancel(); + } + lb.shutdown(); + logger.log(Level.FINE, "Child balancer {0} deleted", key); + } + + private final class ChildLbStateHelper extends ForwardingLoadBalancerHelper { + + @Override + public void updateBalancingState(final ConnectivityState newState, + final SubchannelPicker newPicker) { + // If we are already in the process of resolving addresses, the overall balancing state + // will be updated at the end of it, and we don't need to trigger that update here. + if (!childLbStates.containsKey(key)) { + return; + } + // Subchannel picker and state are saved, but will only be propagated to the channel + // when the child instance exits deactivated state. + currentState = newState; + currentPicker = newPicker; + if (!deactivated && !resolvingAddresses) { + updateOverallBalancingState(); + } + } + + @Override + protected Helper delegate() { + return helper; + } + } + } +} diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java index a640bbd78b9..7257fdfd16b 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java @@ -39,7 +39,6 @@ import io.grpc.xds.XdsClusterResource.CdsUpdate; import io.grpc.xds.XdsClusterResource.CdsUpdate.ClusterType; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; diff --git a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java index b2be811d508..95ca1a33e15 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java @@ -17,7 +17,6 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -46,7 +45,6 @@ import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsNameResolverProvider.CallCounterProvider; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import io.grpc.xds.internal.security.SslContextProviderSupplier; import io.grpc.xds.orca.OrcaPerRequestUtil; import io.grpc.xds.orca.OrcaPerRequestUtil.OrcaPerRequestReportListener; @@ -173,7 +171,7 @@ public void shutdown() { private final class ClusterImplLbHelper extends ForwardingLoadBalancerHelper { private final AtomicLong inFlights; private ConnectivityState currentState = ConnectivityState.IDLE; - private SubchannelPicker currentPicker = BUFFER_PICKER; + private SubchannelPicker currentPicker = LoadBalancer.EMPTY_PICKER; private List dropPolicies = Collections.emptyList(); private long maxConcurrentRequests = DEFAULT_PER_CLUSTER_MAX_CONCURRENT_REQUESTS; @Nullable diff --git a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java index cce32c68246..a4489204236 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java @@ -16,266 +16,63 @@ package io.grpc.xds; -import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.IDLE; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; - -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import io.grpc.ConnectivityState; import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancerProvider; import io.grpc.Status; -import io.grpc.SynchronizationContext; -import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.util.GracefulSwitchLoadBalancer; +import io.grpc.util.MultiChildLoadBalancer; import io.grpc.xds.ClusterManagerLoadBalancerProvider.ClusterManagerConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; /** * The top-level load balancing policy. */ -class ClusterManagerLoadBalancer extends LoadBalancer { - - @VisibleForTesting - static final int DELAYED_CHILD_DELETION_TIME_MINUTES = 15; +class ClusterManagerLoadBalancer extends MultiChildLoadBalancer { - private final Map childLbStates = new HashMap<>(); - private final Helper helper; - private final SynchronizationContext syncContext; - private final ScheduledExecutorService timeService; private final XdsLogger logger; - // Set to true if currently in the process of handling resolved addresses. - private boolean resolvingAddresses; ClusterManagerLoadBalancer(Helper helper) { - this.helper = checkNotNull(helper, "helper"); - this.syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext"); - this.timeService = checkNotNull(helper.getScheduledExecutorService(), "timeService"); + super(helper); logger = XdsLogger.withLogId( InternalLogId.allocate("cluster_manager-lb", helper.getAuthority())); logger.log(XdsLogLevel.INFO, "Created"); } @Override - public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { - try { - resolvingAddresses = true; - return acceptResolvedAddressesInternal(resolvedAddresses); - } finally { - resolvingAddresses = false; - } - } - - public boolean acceptResolvedAddressesInternal(ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); + protected Map getPolicySelectionMap( + ResolvedAddresses resolvedAddresses) { ClusterManagerConfig config = (ClusterManagerConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); - Map newChildPolicies = config.childPolicies; + Map newChildPolicies = new HashMap<>(config.childPolicies); logger.log( XdsLogLevel.INFO, "Received cluster_manager lb config: child names={0}", newChildPolicies.keySet()); - for (Map.Entry entry : newChildPolicies.entrySet()) { - final String name = entry.getKey(); - LoadBalancerProvider childPolicyProvider = entry.getValue().getProvider(); - Object childConfig = entry.getValue().getConfig(); - if (!childLbStates.containsKey(name)) { - childLbStates.put(name, new ChildLbState(name, childPolicyProvider)); - } else { - childLbStates.get(name).reactivate(childPolicyProvider); - } - LoadBalancer childLb = childLbStates.get(name).lb; - ResolvedAddresses childAddresses = - resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(childConfig).build(); - childLb.handleResolvedAddresses(childAddresses); - } - for (String name : childLbStates.keySet()) { - if (!newChildPolicies.containsKey(name)) { - childLbStates.get(name).deactivate(); - } - } - // Must update channel picker before return so that new RPCs will not be routed to deleted - // clusters and resolver can remove them in service config. - updateOverallBalancingState(); - return true; - } - - @Override - public void handleNameResolutionError(Status error) { - logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - boolean gotoTransientFailure = true; - for (ChildLbState state : childLbStates.values()) { - if (!state.deactivated) { - gotoTransientFailure = false; - state.lb.handleNameResolutionError(error); - } - } - if (gotoTransientFailure) { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } + return newChildPolicies; } @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - for (ChildLbState state : childLbStates.values()) { - state.shutdown(); - } - childLbStates.clear(); - } - - private void updateOverallBalancingState() { - ConnectivityState overallState = null; - final Map childPickers = new HashMap<>(); - for (ChildLbState childLbState : childLbStates.values()) { - if (childLbState.deactivated) { - continue; - } - childPickers.put(childLbState.name, childLbState.currentPicker); - overallState = aggregateState(overallState, childLbState.currentState); - } - if (overallState != null) { - SubchannelPicker picker = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - String clusterName = - args.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY); - SubchannelPicker delegate = childPickers.get(clusterName); - if (delegate == null) { - return - PickResult.withError( - Status.UNAVAILABLE.withDescription("CDS encountered error: unable to find " - + "available subchannel for cluster " + clusterName)); - } - return delegate.pickSubchannel(args); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("pickers", childPickers).toString(); - } - }; - helper.updateBalancingState(overallState, picker); - } - } - - @Nullable - private static ConnectivityState aggregateState( - @Nullable ConnectivityState overallState, ConnectivityState childState) { - if (overallState == null) { - return childState; - } - if (overallState == READY || childState == READY) { - return READY; - } - if (overallState == CONNECTING || childState == CONNECTING) { - return CONNECTING; - } - if (overallState == IDLE || childState == IDLE) { - return IDLE; - } - return overallState; - } - - private final class ChildLbState { - private final String name; - private final GracefulSwitchLoadBalancer lb; - private LoadBalancerProvider policyProvider; - private ConnectivityState currentState = CONNECTING; - private SubchannelPicker currentPicker = BUFFER_PICKER; - private boolean deactivated; - @Nullable - ScheduledHandle deletionTimer; - - ChildLbState(String name, LoadBalancerProvider policyProvider) { - this.name = name; - this.policyProvider = policyProvider; - lb = new GracefulSwitchLoadBalancer(new ChildLbStateHelper()); - lb.switchTo(policyProvider); - } - - void deactivate() { - if (deactivated) { - return; - } - - class DeletionTask implements Runnable { - @Override - public void run() { - shutdown(); - childLbStates.remove(name); - } - } - - deletionTimer = - syncContext.schedule( - new DeletionTask(), - DELAYED_CHILD_DELETION_TIME_MINUTES, - TimeUnit.MINUTES, - timeService); - deactivated = true; - logger.log(XdsLogLevel.DEBUG, "Child balancer {0} deactivated", name); - } - - void reactivate(LoadBalancerProvider policyProvider) { - if (deletionTimer != null && deletionTimer.isPending()) { - deletionTimer.cancel(); - deactivated = false; - logger.log(XdsLogLevel.DEBUG, "Child balancer {0} reactivated", name); - } - if (!this.policyProvider.getPolicyName().equals(policyProvider.getPolicyName())) { - logger.log( - XdsLogLevel.DEBUG, - "Child balancer {0} switching policy from {1} to {2}", - name, this.policyProvider.getPolicyName(), policyProvider.getPolicyName()); - lb.switchTo(policyProvider); - this.policyProvider = policyProvider; - } - } - - void shutdown() { - if (deletionTimer != null && deletionTimer.isPending()) { - deletionTimer.cancel(); - } - lb.shutdown(); - logger.log(XdsLogLevel.DEBUG, "Child balancer {0} deleted", name); - } - - private final class ChildLbStateHelper extends ForwardingLoadBalancerHelper { - + protected SubchannelPicker getSubchannelPicker(Map childPickers) { + return new SubchannelPicker() { @Override - public void updateBalancingState(final ConnectivityState newState, - final SubchannelPicker newPicker) { - // If we are already in the process of resolving addresses, the overall balancing state - // will be updated at the end of it, and we don't need to trigger that update here. - if (!childLbStates.containsKey(name)) { - return; - } - // Subchannel picker and state are saved, but will only be propagated to the channel - // when the child instance exits deactivated state. - currentState = newState; - currentPicker = newPicker; - if (!deactivated && !resolvingAddresses) { - updateOverallBalancingState(); + public PickResult pickSubchannel(PickSubchannelArgs args) { + String clusterName = + args.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY); + SubchannelPicker childPicker = childPickers.get(clusterName); + if (childPicker == null) { + return + PickResult.withError( + Status.UNAVAILABLE.withDescription("CDS encountered error: unable to find " + + "available subchannel for cluster " + clusterName)); } + return childPicker.pickSubchannel(args); } @Override - protected Helper delegate() { - return helper; + public String toString() { + return MoreObjects.toStringHelper(this).add("pickers", childPickers).toString(); } - } + }; } } diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java index 3af58ef93cb..a7564e89a8c 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java @@ -55,7 +55,6 @@ import io.grpc.xds.XdsClient.ResourceWatcher; import io.grpc.xds.XdsEndpointResource.EdsUpdate; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; diff --git a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java index e833b3777b8..5cf54317565 100644 --- a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java @@ -21,7 +21,6 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; import io.grpc.ConnectivityState; import io.grpc.InternalLogId; @@ -36,7 +35,6 @@ import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -149,7 +147,7 @@ private void tryNextPriority() { ChildLbState child = new ChildLbState(priority, priorityConfigs.get(priority).ignoreReresolution); children.put(priority, child); - updateOverallState(priority, CONNECTING, BUFFER_PICKER); + updateOverallState(priority, CONNECTING, LoadBalancer.EMPTY_PICKER); // Calling the child's updateResolvedAddresses() can result in tryNextPriority() being // called recursively. We need to be sure to be done with processing here before it is // called. @@ -210,7 +208,7 @@ private final class ChildLbState { @Nullable ScheduledHandle deletionTimer; @Nullable String policy; ConnectivityState connectivityState = CONNECTING; - SubchannelPicker picker = BUFFER_PICKER; + SubchannelPicker picker = LoadBalancer.EMPTY_PICKER; ChildLbState(final String priority, boolean ignoreReresolution) { this.priority = priority; diff --git a/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java b/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java index 436eca8ec5d..20a70cb0322 100644 --- a/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java @@ -39,7 +39,6 @@ import io.grpc.Status; import io.grpc.SynchronizationContext; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.SocketAddress; import java.util.ArrayList; import java.util.Collections; diff --git a/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java b/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java index 825e4a8eca0..596247b8234 100644 --- a/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java @@ -21,7 +21,6 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; import com.google.common.collect.ImmutableMap; import io.grpc.ConnectivityState; @@ -34,7 +33,6 @@ import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -159,7 +157,7 @@ private void updateOverallBalancingState() { if (overallState == TRANSIENT_FAILURE) { picker = new WeightedRandomPicker(errorPickers); } else { - picker = XdsSubchannelPickers.BUFFER_PICKER; + picker = LoadBalancer.EMPTY_PICKER; } } else { picker = new WeightedRandomPicker(childPickers); @@ -191,7 +189,7 @@ private static ConnectivityState aggregateState( private final class ChildHelper extends ForwardingLoadBalancerHelper { String name; ConnectivityState currentState = CONNECTING; - SubchannelPicker currentPicker = BUFFER_PICKER; + SubchannelPicker currentPicker = LoadBalancer.EMPTY_PICKER; private ChildHelper(String name) { this.name = name; diff --git a/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java b/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java index b9196492624..885844f1cbf 100644 --- a/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java @@ -32,7 +32,6 @@ import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java b/xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java deleted file mode 100644 index 5c2890c34ea..00000000000 --- a/xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.base.MoreObjects; -import io.grpc.LoadBalancer.PickResult; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.Status; - -final class XdsSubchannelPickers { - - private XdsSubchannelPickers() { /* DO NOT CALL ME */ } - - static final SubchannelPicker BUFFER_PICKER = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withNoResult(); - } - - @Override - public String toString() { - return "BUFFER_PICKER"; - } - }; - - static final class ErrorPicker extends SubchannelPicker { - - private final Status error; - - ErrorPicker(Status error) { - this.error = checkNotNull(error, "error"); - } - - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withError(error); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("error", error) - .toString(); - } - } -} diff --git a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java index 6cb12550b60..6eab6151477 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java @@ -54,7 +54,6 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.testing.TestMethodDescriptors; import io.grpc.xds.ClusterManagerLoadBalancerProvider.ClusterManagerConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -126,10 +125,14 @@ public void handleResolvedAddressesUpdatesChannelPicker() { assertThat(pickSubchannel(picker, "childA")).isEqualTo(PickResult.withNoResult()); assertThat(pickSubchannel(picker, "childB")).isEqualTo(PickResult.withNoResult()); assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); - assertThat(childBalancer1.name).isEqualTo("policy_a"); - assertThat(childBalancer2.name).isEqualTo("policy_b"); + assertThat(childBalancers.stream() + .filter(b -> b.name.equals("policy_a")) + .count()).isEqualTo(1); + assertThat(childBalancers.stream() + .filter(b -> b.name.equals("policy_b")) + .count()).isEqualTo(1); + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); assertThat(childBalancer1.config).isEqualTo(lbConfigInventory.get("childA")); assertThat(childBalancer2.config).isEqualTo(lbConfigInventory.get("childB")); @@ -151,8 +154,7 @@ public void handleResolvedAddressesUpdatesChannelPicker() { assertThat(childBalancer2.shutdown).isFalse(); assertThat(childBalancers).hasSize(3); - FakeLoadBalancer childBalancer3 = childBalancers.get(2); - assertThat(childBalancer3.name).isEqualTo("policy_c"); + FakeLoadBalancer childBalancer3 = getChildBalancerByName("policy_c"); assertThat(childBalancer3.config).isEqualTo(lbConfigInventory.get("childC")); // delayed policy_b deletion @@ -166,8 +168,8 @@ public void updateBalancingStateFromChildBalancers() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); Subchannel subchannel1 = mock(Subchannel.class); Subchannel subchannel2 = mock(Subchannel.class); childBalancer1.deliverSubchannelState(subchannel1, ConnectivityState.READY); @@ -184,11 +186,20 @@ public void updateBalancingStateFromChildBalancers() { .isEqualTo(subchannel2); } + private FakeLoadBalancer getChildBalancerByName(String name) { + for (FakeLoadBalancer childLb : childBalancers) { + if (childLb.name.equals(name)) { + return childLb; + } + } + return null; + } + @Test public void ignoreBalancingStateUpdateForDeactivatedChildLbs() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); deliverResolvedAddresses(ImmutableMap.of("childB", "policy_b")); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); // policy_a (deactivated) + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); // policy_a (deactivated) Subchannel subchannel = mock(Subchannel.class); childBalancer1.deliverSubchannelState(subchannel, ConnectivityState.READY); verify(helper, never()).updateBalancingState( @@ -231,8 +242,8 @@ public void handleNameResolutionError_beforeChildLbsInstantiated_returnErrorPick public void handleNameResolutionError_afterChildLbsInstantiated_propagateToChildLbs() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); clusterManagerLoadBalancer.handleNameResolutionError( Status.UNAVAILABLE.withDescription("resolver error")); assertThat(childBalancer1.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); @@ -245,8 +256,8 @@ public void handleNameResolutionError_afterChildLbsInstantiated_propagateToChild public void handleNameResolutionError_notPropagateToDeactivatedChildLbs() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); deliverResolvedAddresses(ImmutableMap.of("childB", "policy_b")); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); // policy_a (deactivated) - FakeLoadBalancer childBalancer2 = childBalancers.get(1); // policy_b + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); // policy_a (deactivated) + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); // policy_b clusterManagerLoadBalancer.handleNameResolutionError( Status.UNKNOWN.withDescription("unknown error")); assertThat(childBalancer1.upstreamError).isNull(); diff --git a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java index a005f40fad7..ebcd68dc950 100644 --- a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java @@ -21,7 +21,7 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; +import static io.grpc.LoadBalancer.EMPTY_PICKER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; @@ -41,6 +41,7 @@ import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.ErrorPicker; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; @@ -55,7 +56,6 @@ import io.grpc.internal.TestUtils.StandardLoadBalancerProvider; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.ArrayList; @@ -423,13 +423,13 @@ public void idleToConnectingDoesNotTriggerFailOver() { // p0 gets IDLE. helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 goes to CONNECTING helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // no failover happened @@ -459,15 +459,15 @@ public void connectingResetFailOverIfSeenReadyOrIdleSinceTransientFailure() { // p0 gets IDLE. helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 goes to CONNECTING, reset failover timer fakeClock.forwardTime(5, TimeUnit.SECONDS); helper0.updateBalancingState( CONNECTING, - BUFFER_PICKER); - verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + EMPTY_PICKER); + verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // failover happens fakeClock.forwardTime(10, TimeUnit.SECONDS); @@ -509,7 +509,7 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { // p0 goes to CONNECTING helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // no failover happened @@ -560,7 +560,7 @@ public void typicalPriorityFailOverFlowWithIdleUpdate() { // p0 gets IDLE. helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 fails over to p1 immediately. @@ -581,13 +581,13 @@ public void typicalPriorityFailOverFlowWithIdleUpdate() { // p2 gets IDLE helper2.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 gets back to IDLE helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p2 fails but does not affect overall picker @@ -614,13 +614,13 @@ public void typicalPriorityFailOverFlowWithIdleUpdate() { // p2 gets back to IDLE helper2.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 gets back to IDLE helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 fails over to p2 and picker is updated to p2's existing picker. diff --git a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java index 91ab1e8fac4..6cec8b0fb6f 100644 --- a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java @@ -20,7 +20,7 @@ import static io.grpc.ConnectivityState.CONNECTING; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; +import static io.grpc.LoadBalancer.EMPTY_PICKER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeast; @@ -39,6 +39,7 @@ import io.grpc.Attributes; import io.grpc.EquivalentAddressGroup; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.ErrorPicker; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; @@ -52,7 +53,6 @@ import io.grpc.xds.WeightedRandomPicker.WeightedChildPicker; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.ArrayList; @@ -209,7 +209,7 @@ public void handleResolvedAddresses() { .setAttributes(Attributes.newBuilder().set(fakeKey, fakeValue).build()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); assertThat(childBalancers).hasSize(4); assertThat(childHelpers).hasSize(4); assertThat(fooLbCreated).isEqualTo(2); @@ -246,7 +246,7 @@ public void handleResolvedAddresses() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(newTargets)) .build()); - verify(helper, atLeast(2)).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper, atLeast(2)).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); assertThat(childBalancers).hasSize(5); assertThat(childHelpers).hasSize(5); assertThat(fooLbCreated).isEqualTo(3); // One more foo LB created for target4 @@ -288,7 +288,7 @@ public void handleNameResolutionError() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // Error after child balancers created. weightedTargetLb.handleNameResolutionError(Status.ABORTED); @@ -315,7 +315,7 @@ public void balancingStateUpdatedFromChildBalancers() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // Subchannels to be created for each child balancer. final SubchannelPicker[] subchannelPickers = new SubchannelPicker[]{ @@ -335,7 +335,7 @@ public void balancingStateUpdatedFromChildBalancers() { childHelpers.get(1).updateBalancingState(TRANSIENT_FAILURE, failurePickers[1]); verify(helper, never()).updateBalancingState( eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // Another child balancer goes to READY. childHelpers.get(2).updateBalancingState(READY, subchannelPickers[2]); @@ -396,7 +396,7 @@ public void raceBetweenShutdownAndChildLbBalancingStateUpdate() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // LB shutdown and subchannel state change can happen simultaneously. If shutdown runs first, // any further balancing state update should be ignored. diff --git a/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java index 344876aa348..bb80635f1bd 100644 --- a/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java @@ -30,6 +30,7 @@ import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.ErrorPicker; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.ResolvedAddresses; import io.grpc.LoadBalancer.SubchannelPicker; @@ -41,7 +42,6 @@ import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; import io.grpc.xds.WrrLocalityLoadBalancer.WrrLocalityConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.SocketAddress; import java.util.Collections; import java.util.List; From 3b61799f73dc9f52488476c4c92b9e7ce978dc2e Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 9 Aug 2023 09:44:18 -0700 Subject: [PATCH 04/14] okhttp: Add OkHttpServerProvider This allows okhttp to service the Grpc.newServerBuilderForPort() API. Note that, unlike Netty, it will throw if you try to use ServerBuilder.forPort(). This fixes android-interop-testing which was broken by c9864a119. --- api/src/main/java/io/grpc/ServerRegistry.java | 17 ++++- .../java/io/grpc/ServerRegistryAccessor.java | 26 +++++++ okhttp/build.gradle | 1 + .../io/grpc/okhttp/OkHttpServerBuilder.java | 4 +- .../io/grpc/okhttp/OkHttpServerProvider.java | 55 ++++++++++++++ .../META-INF/services/io.grpc.ServerProvider | 1 + .../grpc/okhttp/OkHttpServerProviderTest.java | 71 +++++++++++++++++++ 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java create mode 100644 okhttp/src/main/java/io/grpc/okhttp/OkHttpServerProvider.java create mode 100644 okhttp/src/main/resources/META-INF/services/io.grpc.ServerProvider create mode 100644 okhttp/src/test/java/io/grpc/okhttp/OkHttpServerProviderTest.java diff --git a/api/src/main/java/io/grpc/ServerRegistry.java b/api/src/main/java/io/grpc/ServerRegistry.java index e6a067ce87f..70fd3657307 100644 --- a/api/src/main/java/io/grpc/ServerRegistry.java +++ b/api/src/main/java/io/grpc/ServerRegistry.java @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -92,7 +93,7 @@ public static synchronized ServerRegistry getDefaultRegistry() { if (instance == null) { List providerList = ServiceProviders.loadAll( ServerProvider.class, - Collections.>emptyList(), + getHardCodedClasses(), ServerProvider.class.getClassLoader(), new ServerPriorityAccessor()); instance = new ServerRegistry(); @@ -119,6 +120,20 @@ ServerProvider provider() { return providers.isEmpty() ? null : providers.get(0); } + @VisibleForTesting + static List> getHardCodedClasses() { + // Class.forName(String) is used to remove the need for ProGuard configuration. Note that + // ProGuard does not detect usages of Class.forName(String, boolean, ClassLoader): + // https://sourceforge.net/p/proguard/bugs/418/ + List> list = new ArrayList<>(); + try { + list.add(Class.forName("io.grpc.okhttp.OkHttpServerProvider")); + } catch (ClassNotFoundException e) { + logger.log(Level.FINE, "Unable to find OkHttpServerProvider", e); + } + return Collections.unmodifiableList(list); + } + ServerBuilder newServerBuilderForPort(int port, ServerCredentials creds) { List providers = providers(); if (providers.isEmpty()) { diff --git a/api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java b/api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java new file mode 100644 index 00000000000..b15d4dfde19 --- /dev/null +++ b/api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc; + +/** Accesses test-only methods of {@link ServerRegistry}. */ +public final class ServerRegistryAccessor { + private ServerRegistryAccessor() {} + + public static Iterable> getHardCodedClasses() { + return ServerRegistry.getHardCodedClasses(); + } +} diff --git a/okhttp/build.gradle b/okhttp/build.gradle index 7d84df436db..99f799ec97f 100644 --- a/okhttp/build.gradle +++ b/okhttp/build.gradle @@ -50,6 +50,7 @@ tasks.named("checkstyleMain").configure { tasks.named("javadoc").configure { options.links 'http://square.github.io/okhttp/2.x/okhttp/' exclude 'io/grpc/okhttp/Internal*' + exclude 'io/grpc/okhttp/*Provider.java' exclude 'io/grpc/okhttp/internal/**' } diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java index 45d6b9efc54..8269a8ddf0f 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.DoNotCall; @@ -89,7 +88,7 @@ public final class OkHttpServerBuilder extends ForwardingServerBuilder provider.builderForPort(80)); + } + + @Test + public void newServerBuilderForPort_success() { + ServerProvider.NewServerBuilderResult result = + provider.newServerBuilderForPort(80, InsecureServerCredentials.create()); + assertThat(result.getServerBuilder()).isInstanceOf(OkHttpServerBuilder.class); + } + + @Test + public void newServerBuilderForPort_fail() { + ServerProvider.NewServerBuilderResult result = provider.newServerBuilderForPort( + 80, new FakeServerCredentials()); + assertThat(result.getError()).contains("FakeServerCredentials"); + } + + private static final class FakeServerCredentials extends ServerCredentials {} +} From cebb4659a1b3b5d484749f4c488ddde470bcccc0 Mon Sep 17 00:00:00 2001 From: Mohan Li <67390330+mohanli-ml@users.noreply.github.com> Date: Thu, 10 Aug 2023 10:30:41 -0700 Subject: [PATCH 05/14] test: allow set request/response size in interop soak test (#10465) --- .../integration/AbstractInteropTest.java | 18 +++++++++++------ .../integration/TestServiceClient.java | 20 +++++++++++++++++-- .../integration/XdsFederationTestClient.java | 20 ++++++++++++++++++- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java index 82c49b5813b..48001cbdc1f 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java @@ -2007,18 +2007,21 @@ public Status getStatus() { } private SoakIterationResult performOneSoakIteration( - TestServiceGrpc.TestServiceBlockingStub soakStub) throws Exception { + TestServiceGrpc.TestServiceBlockingStub soakStub, int soakRequestSize, int soakResponseSize) + throws Exception { long startNs = System.nanoTime(); Status status = Status.OK; try { final SimpleRequest request = SimpleRequest.newBuilder() - .setResponseSize(314159) - .setPayload(Payload.newBuilder().setBody(ByteString.copyFrom(new byte[271828]))) + .setResponseSize(soakResponseSize) + .setPayload( + Payload.newBuilder().setBody(ByteString.copyFrom(new byte[soakRequestSize]))) .build(); final SimpleResponse goldenResponse = SimpleResponse.newBuilder() - .setPayload(Payload.newBuilder().setBody(ByteString.copyFrom(new byte[314159]))) + .setPayload( + Payload.newBuilder().setBody(ByteString.copyFrom(new byte[soakResponseSize]))) .build(); assertResponse(goldenResponse, soakStub.unaryCall(request)); } catch (StatusRuntimeException e) { @@ -2039,7 +2042,9 @@ public void performSoakTest( int maxFailures, int maxAcceptablePerIterationLatencyMs, int minTimeMsBetweenRpcs, - int overallTimeoutSeconds) + int overallTimeoutSeconds, + int soakRequestSize, + int soakResponseSize) throws Exception { int iterationsDone = 0; int totalFailures = 0; @@ -2063,7 +2068,8 @@ public void performSoakTest( .newBlockingStub(soakChannel) .withInterceptors(recordClientCallInterceptor(clientCallCapture)); } - SoakIterationResult result = performOneSoakIteration(soakStub); + SoakIterationResult result = + performOneSoakIteration(soakStub, soakRequestSize, soakResponseSize); SocketAddress peer = clientCallCapture .get().getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); StringBuilder logStr = new StringBuilder( diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java index a6e2c4f3bf8..768a32be62c 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java @@ -97,6 +97,8 @@ public static void main(String[] args) throws Exception { private int soakMinTimeMsBetweenRpcs = 0; private int soakOverallTimeoutSeconds = soakIterations * soakPerIterationMaxAcceptableLatencyMs / 1000; + private int soakRequestSize = 271828; + private int soakResponseSize = 314159; private String additionalMetadata = ""; private static LoadBalancerProvider customBackendMetricsLoadBalancerProvider; @@ -175,6 +177,10 @@ void parseArgs(String[] args) throws Exception { soakMinTimeMsBetweenRpcs = Integer.parseInt(value); } else if ("soak_overall_timeout_seconds".equals(key)) { soakOverallTimeoutSeconds = Integer.parseInt(value); + } else if ("soak_request_size".equals(key)) { + soakRequestSize = Integer.parseInt(value); + } else if ("soak_response_size".equals(key)) { + soakResponseSize = Integer.parseInt(value); } else if ("additional_metadata".equals(key)) { additionalMetadata = value; } else { @@ -247,6 +253,12 @@ void parseArgs(String[] args) throws Exception { + "\n should stop and fail, if the desired number of " + "\n iterations have not yet completed. Default " + c.soakOverallTimeoutSeconds + + "\n --soak_request_size " + + "\n The request size in a soak RPC. Default " + + c.soakRequestSize + + "\n --soak_response_size " + + "\n The response size in a soak RPC. Default " + + c.soakResponseSize + "\n --additional_metadata " + "\n Additional metadata to send in each request, as a " + "\n semicolon-separated list of key:value pairs. Default " @@ -481,7 +493,9 @@ private void runTest(TestCases testCase) throws Exception { soakMaxFailures, soakPerIterationMaxAcceptableLatencyMs, soakMinTimeMsBetweenRpcs, - soakOverallTimeoutSeconds); + soakOverallTimeoutSeconds, + soakRequestSize, + soakResponseSize); break; } @@ -493,7 +507,9 @@ private void runTest(TestCases testCase) throws Exception { soakMaxFailures, soakPerIterationMaxAcceptableLatencyMs, soakMinTimeMsBetweenRpcs, - soakOverallTimeoutSeconds); + soakOverallTimeoutSeconds, + soakRequestSize, + soakResponseSize); break; } diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java index 8f166b6affa..9b01df0d18d 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java @@ -74,6 +74,8 @@ public void run() { private int soakPerIterationMaxAcceptableLatencyMs = 1000; private int soakOverallTimeoutSeconds = 10; private int soakMinTimeMsBetweenRpcs = 0; + private int soakRequestSize = 271828; + private int soakResponseSize = 314159; private String testCase = "rpc_soak"; private final ArrayList clients = new ArrayList<>(); @@ -122,6 +124,12 @@ private void parseArgs(String[] args) { case "soak_min_time_ms_between_rpcs": soakMinTimeMsBetweenRpcs = Integer.parseInt(value); break; + case "soak_request_size": + soakRequestSize = Integer.parseInt(value); + break; + case "soak_response_size": + soakResponseSize = Integer.parseInt(value); + break; default: System.err.println("Unknown argument: " + key); usage = true; @@ -175,6 +183,14 @@ private void parseArgs(String[] args) { + "\n channel_soak: sends --soak_iterations RPCs, rebuilding the channel " + "each time." + "\n Default: " + c.testCase + + "\n --soak_request_size " + + "\n The request size in a soak RPC. Default " + + c.soakRequestSize + + "\n" + + " --soak_response_size \n" + + " The response size in a soak RPC. Default" + + " " + + c.soakResponseSize ); System.exit(1); } @@ -249,7 +265,9 @@ public void run() { soakMaxFailures, soakPerIterationMaxAcceptableLatencyMs, soakMinTimeMsBetweenRpcs, - soakOverallTimeoutSeconds); + soakOverallTimeoutSeconds, + soakRequestSize, + soakResponseSize); logger.info("Test case: " + testCase + " done for server: " + serverUri); runSucceeded = true; } catch (Exception e) { From f8baa9ca1dc02559686fe69bc5442c596597971c Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 10 Aug 2023 11:02:12 -0700 Subject: [PATCH 06/14] Upgrade protobuf-java and protoc to 3.24.0 --- examples/android/clientcache/app/build.gradle | 2 +- examples/android/helloworld/app/build.gradle | 2 +- examples/android/routeguide/app/build.gradle | 2 +- examples/android/strictmode/app/build.gradle | 2 +- examples/build.gradle | 2 +- examples/example-alts/build.gradle | 2 +- examples/example-debug/build.gradle | 2 +- examples/example-debug/pom.xml | 2 +- examples/example-gauth/build.gradle | 2 +- examples/example-gauth/pom.xml | 2 +- examples/example-gcp-observability/build.gradle | 2 +- examples/example-hostname/build.gradle | 2 +- examples/example-hostname/pom.xml | 2 +- examples/example-jwt-auth/build.gradle | 2 +- examples/example-jwt-auth/pom.xml | 4 ++-- examples/example-orca/build.gradle | 2 +- examples/example-reflection/build.gradle | 2 +- examples/example-servlet/build.gradle | 2 +- examples/example-tls/build.gradle | 2 +- examples/example-tls/pom.xml | 2 +- examples/example-xds/build.gradle | 2 +- examples/pom.xml | 4 ++-- gradle/libs.versions.toml | 2 +- repositories.bzl | 12 ++++++------ 24 files changed, 31 insertions(+), 31 deletions(-) diff --git a/examples/android/clientcache/app/build.gradle b/examples/android/clientcache/app/build.gradle index 1a481266956..e2fead073a0 100644 --- a/examples/android/clientcache/app/build.gradle +++ b/examples/android/clientcache/app/build.gradle @@ -32,7 +32,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index 4cc2100d23c..0ec3da6410c 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -30,7 +30,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index acadd301b51..92af8f3706d 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -30,7 +30,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/strictmode/app/build.gradle b/examples/android/strictmode/app/build.gradle index e8eb82e1373..031140f16ce 100644 --- a/examples/android/strictmode/app/build.gradle +++ b/examples/android/strictmode/app/build.gradle @@ -31,7 +31,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/build.gradle b/examples/build.gradle index 79ac3074aaf..fb799b42eda 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -24,7 +24,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' def protocVersion = protobufVersion dependencies { diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index 7830538a766..953e53a829c 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { // grpc-alts transitively depends on grpc-netty-shaded, grpc-protobuf, and grpc-stub diff --git a/examples/example-debug/build.gradle b/examples/example-debug/build.gradle index 84463a0ed44..90922ab1f5e 100644 --- a/examples/example-debug/build.gradle +++ b/examples/example-debug/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-debug/pom.xml b/examples/example-debug/pom.xml index 95ac88c813f..71d085d1559 100644 --- a/examples/example-debug/pom.xml +++ b/examples/example-debug/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index c2b23d41a90..2f1a0a8a3b1 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' def protocVersion = protobufVersion diff --git a/examples/example-gauth/pom.xml b/examples/example-gauth/pom.xml index 3edbf030179..a92db60b1a9 100644 --- a/examples/example-gauth/pom.xml +++ b/examples/example-gauth/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-gcp-observability/build.gradle b/examples/example-gcp-observability/build.gradle index 67242246305..9df847277dd 100644 --- a/examples/example-gcp-observability/build.gradle +++ b/examples/example-gcp-observability/build.gradle @@ -27,7 +27,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-hostname/build.gradle b/examples/example-hostname/build.gradle index bd60e8b77d2..e940e756801 100644 --- a/examples/example-hostname/build.gradle +++ b/examples/example-hostname/build.gradle @@ -24,7 +24,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-hostname/pom.xml b/examples/example-hostname/pom.xml index 1cec5319423..422bc6a554c 100644 --- a/examples/example-hostname/pom.xml +++ b/examples/example-hostname/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index d6ebdb0f161..3964e0a6276 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -25,7 +25,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' def protocVersion = protobufVersion dependencies { diff --git a/examples/example-jwt-auth/pom.xml b/examples/example-jwt-auth/pom.xml index 0ed7b4cc032..d955518a8cf 100644 --- a/examples/example-jwt-auth/pom.xml +++ b/examples/example-jwt-auth/pom.xml @@ -14,8 +14,8 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 - 3.23.4 + 3.24.0 + 3.24.0 1.8 1.8 diff --git a/examples/example-orca/build.gradle b/examples/example-orca/build.gradle index 852403fe240..c32e5ca03b5 100644 --- a/examples/example-orca/build.gradle +++ b/examples/example-orca/build.gradle @@ -20,7 +20,7 @@ java { } def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-reflection/build.gradle b/examples/example-reflection/build.gradle index c4499953827..2b9971981d5 100644 --- a/examples/example-reflection/build.gradle +++ b/examples/example-reflection/build.gradle @@ -20,7 +20,7 @@ java { } def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-servlet/build.gradle b/examples/example-servlet/build.gradle index 1563a88633a..54e65ead77e 100644 --- a/examples/example-servlet/build.gradle +++ b/examples/example-servlet/build.gradle @@ -18,7 +18,7 @@ java { } def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}", diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index ffa42d8bfe2..1144bf04c6c 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-tls/pom.xml b/examples/example-tls/pom.xml index 371d3bce03f..cd0cb87d2a2 100644 --- a/examples/example-tls/pom.xml +++ b/examples/example-tls/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-xds/build.gradle b/examples/example-xds/build.gradle index 9222ce2d7fc..74175c2ba07 100644 --- a/examples/example-xds/build.gradle +++ b/examples/example-xds/build.gradle @@ -25,7 +25,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/pom.xml b/examples/pom.xml index 793604cd804..3563d599851 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -13,8 +13,8 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 - 3.23.4 + 3.24.0 + 3.24.0 1.8 1.8 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5a248478f08..9ed986c9aa2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ netty = '4.1.93.Final' # SECURITY.md nettytcnative = '2.0.61.Final' opencensus = "0.31.1" -protobuf = "3.23.4" +protobuf = "3.24.0" [libraries] android-annotations = "com.google.android:annotations:4.1.1.4" diff --git a/repositories.bzl b/repositories.bzl index ce86fbd68c0..d9a383a3e89 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -143,18 +143,18 @@ def com_google_protobuf(): # This statement defines the @com_google_protobuf repo. http_archive( name = "com_google_protobuf", - sha256 = "ac3fd4e97af55405d8bfba43c22d8a7e464a371bb6bc9e706627b745c1022dbf", - strip_prefix = "protobuf-23.4", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protobuf-23.4.zip"], + sha256 = "5980276108f948e1ada091475549a8c75dc83c193129aab0e986ceaac3e97131", + strip_prefix = "protobuf-24.0", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protobuf-24.0.zip"], ) def com_google_protobuf_javalite(): # java_lite_proto_library rules implicitly depend on @com_google_protobuf_javalite http_archive( name = "com_google_protobuf_javalite", - sha256 = "ac3fd4e97af55405d8bfba43c22d8a7e464a371bb6bc9e706627b745c1022dbf", - strip_prefix = "protobuf-23.4", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protobuf-23.4.zip"], + sha256 = "5980276108f948e1ada091475549a8c75dc83c193129aab0e986ceaac3e97131", + strip_prefix = "protobuf-24.0", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protobuf-24.0.zip"], ) def io_grpc_grpc_proto(): From 47a84c48c7da64e1708f8e391454a870a557e48e Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 2 Aug 2023 13:51:56 -0700 Subject: [PATCH 07/14] RELEASING.md: Prefix grpc/grpc commit with [interop] They require this style. --- RELEASING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASING.md b/RELEASING.md index 17ea03bff94..203daec4aa6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -174,7 +174,7 @@ Tagging the Release tools/interop_matrix/testcases/java__master # Commit the changes - git commit --all -m "Add grpc-java $MAJOR.$MINOR.$PATCH to client_matrix.py" + git commit --all -m "[interop] Add grpc-java $MAJOR.$MINOR.$PATCH to client_matrix.py" # Create a PR with the `release notes: no` label and run ad-hoc test against your PR ``` From 778c209751bf4d3ca3f5437ba8089d7bddc5b6bc Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Fri, 8 Jul 2022 15:18:21 -0700 Subject: [PATCH 08/14] java_grpc_library.bzl: Allow toolchain to use annotation processors While not easy to use because java_grpc_library() uses a fixed toolchain, it is possible for downstream users to apply a patch to the repo to add their own annotation processors. This feature was added inside Google so exporting it reduces the diff between internal and external and causes no harm. cl/280287611 --- java_grpc_library.bzl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/java_grpc_library.bzl b/java_grpc_library.bzl index 4c7a7f2791b..88ad8b889a3 100644 --- a/java_grpc_library.bzl +++ b/java_grpc_library.bzl @@ -3,6 +3,7 @@ _JavaRpcToolchainInfo = provider( fields = [ "java_toolchain", + "java_plugins", "plugin", "plugin_arg", "protoc", @@ -14,6 +15,7 @@ def _java_rpc_toolchain_impl(ctx): return [ _JavaRpcToolchainInfo( java_toolchain = ctx.attr._java_toolchain, + java_plugins = ctx.attr.java_plugins, plugin = ctx.executable.plugin, plugin_arg = ctx.attr.plugin_arg, protoc = ctx.executable._protoc, @@ -39,6 +41,10 @@ java_rpc_toolchain = rule( default = Label("@com_google_protobuf//:protoc"), executable = True, ), + "java_plugins": attr.label_list( + default = [], + providers = [JavaPluginInfo], + ), "_java_toolchain": attr.label( default = Label("@bazel_tools//tools/jdk:current_java_toolchain"), ), @@ -104,6 +110,7 @@ def _java_rpc_library_impl(ctx): source_jars = [srcjar], output = ctx.outputs.jar, output_source_jar = ctx.outputs.srcjar, + plugins = [plugin[JavaPluginInfo] for plugin in toolchain.java_plugins], deps = [ java_common.make_non_strict(deps_java_info), ] + [dep[JavaInfo] for dep in toolchain.runtime], From fba7835de1d57664d9e3434707988cc8ae7aa80a Mon Sep 17 00:00:00 2001 From: Tony An Date: Mon, 14 Aug 2023 15:23:40 -0700 Subject: [PATCH 09/14] new pick first policy, architectural change (#10354) --- .../internal/PickFirstLeafLoadBalancer.java | 477 ++++ .../PickFirstLeafLoadBalancerTest.java | 2131 +++++++++++++++++ 2 files changed, 2608 insertions(+) create mode 100644 core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java create mode 100644 core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java diff --git a/core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java b/core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java new file mode 100644 index 00000000000..1ea61252e6b --- /dev/null +++ b/core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java @@ -0,0 +1,477 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.IDLE; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.SHUTDOWN; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Lists; +import io.grpc.Attributes; +import io.grpc.ConnectivityState; +import io.grpc.ConnectivityStateInfo; +import io.grpc.EquivalentAddressGroup; +import io.grpc.ExperimentalApi; +import io.grpc.LoadBalancer; +import io.grpc.Status; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + +/** + * A {@link LoadBalancer} that provides no load-balancing over the addresses from the {@link + * io.grpc.NameResolver}. The channel's default behavior is used, which is walking down the address + * list and sticking to the first that works. + */ +@ExperimentalApi("https://github.com/grpc/grpc-java/issues/10383") +final class PickFirstLeafLoadBalancer extends LoadBalancer { + private final Helper helper; + private final Map subchannels = new HashMap<>(); + private Index addressIndex; + private ConnectivityState currentState = IDLE; + + PickFirstLeafLoadBalancer(Helper helper) { + this.helper = checkNotNull(helper, "helper"); + } + + @Override + public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { + List servers = resolvedAddresses.getAddresses(); + if (servers.isEmpty()) { + handleNameResolutionError(Status.UNAVAILABLE.withDescription( + "NameResolver returned no usable address. addrs=" + resolvedAddresses.getAddresses() + + ", attrs=" + resolvedAddresses.getAttributes())); + return false; + } + for (EquivalentAddressGroup eag : servers) { + if (eag == null) { + handleNameResolutionError(Status.UNAVAILABLE.withDescription( + "NameResolver returned address list with null endpoint. addrs=" + + resolvedAddresses.getAddresses() + ", attrs=" + resolvedAddresses.getAttributes())); + return false; + } + } + // We can optionally be configured to shuffle the address list. This can help better distribute + // the load. + if (resolvedAddresses.getLoadBalancingPolicyConfig() + instanceof PickFirstLeafLoadBalancerConfig) { + PickFirstLeafLoadBalancerConfig config + = (PickFirstLeafLoadBalancerConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); + if (config.shuffleAddressList != null && config.shuffleAddressList) { + servers = new ArrayList(servers); + Collections.shuffle(servers, + config.randomSeed != null ? new Random(config.randomSeed) : new Random()); + } + } + + final List newImmutableAddressGroups = + Collections.unmodifiableList(new ArrayList<>(servers)); + + if (addressIndex == null) { + addressIndex = new Index(newImmutableAddressGroups); + } else if (currentState == READY) { + // If a ready subchannel exists in new address list, + // keep this connection and don't create new subchannels + SocketAddress previousAddress = addressIndex.getCurrentAddress(); + addressIndex.updateGroups(newImmutableAddressGroups); + if (addressIndex.seekTo(previousAddress)) { + return true; + } + addressIndex.reset(); + } else { + addressIndex.updateGroups(newImmutableAddressGroups); + } + + // Create subchannels for all new addresses, preserving existing connections + Set oldAddrs = new HashSet<>(subchannels.keySet()); + Set newAddrs = new HashSet<>(); + for (EquivalentAddressGroup endpoint : newImmutableAddressGroups) { + for (SocketAddress addr : endpoint.getAddresses()) { + newAddrs.add(addr); + if (!subchannels.containsKey(addr)) { + createNewSubchannel(addr); + } + } + } + + // remove old subchannels that were not in new address list + for (SocketAddress oldAddr : oldAddrs) { + if (!newAddrs.contains(oldAddr)) { + subchannels.get(oldAddr).getSubchannel().shutdown(); + subchannels.remove(oldAddr); + } + } + + if (oldAddrs.size() == 0 || currentState == CONNECTING || currentState == READY) { + // start connection attempt at first address + updateBalancingState(CONNECTING, new Picker(PickResult.withNoResult())); + requestConnection(); + + } else if (currentState == IDLE) { + // start connection attempt at first address when requested + SubchannelPicker picker = new RequestConnectionPicker(this); + updateBalancingState(IDLE, picker); + + } else if (currentState == TRANSIENT_FAILURE) { + // start connection attempt at first address + requestConnection(); + } + + return true; + } + + @Override + public void handleNameResolutionError(Status error) { + for (SubchannelData subchannelData : subchannels.values()) { + subchannelData.getSubchannel().shutdown(); + } + subchannels.clear(); + // NB(lukaszx0) Whether we should propagate the error unconditionally is arguable. It's fine + // for time being. + updateBalancingState(TRANSIENT_FAILURE, new Picker(PickResult.withError(error))); + } + + void processSubchannelState(Subchannel subchannel, ConnectivityStateInfo stateInfo) { + ConnectivityState newState = stateInfo.getState(); + // Shutdown channels/previously relevant subchannels can still callback with state updates. + // To prevent pickers from returning these obselete subchannels, this logic + // is included to check if the current list of active subchannels includes this subchannel. + if (!subchannels.containsKey(getAddress(subchannel)) + || subchannels.get(getAddress(subchannel)).getSubchannel() != subchannel) { + return; + } + if (newState == SHUTDOWN) { + return; + } + if (newState == IDLE) { + helper.refreshNameResolution(); + } + // If we are transitioning from a TRANSIENT_FAILURE to CONNECTING or IDLE we ignore this state + // transition and still keep the LB in TRANSIENT_FAILURE state. This is referred to as "sticky + // transient failure". Only a subchannel state change to READY will get the LB out of + // TRANSIENT_FAILURE. If the state is IDLE we additionally request a new connection so that we + // keep retrying for a connection. + + // With the new pick first implementation, individual subchannels will have their own backoff + // on a per-address basis. Thus, iterative requests for connections will not be requested + // once the first pass through is complete. + // However, every time there is an address update, we will perform a pass through for the new + // addresses in the updated list. + subchannels.get(getAddress(subchannel)).updateState(newState); + if (currentState == TRANSIENT_FAILURE) { + if (newState == CONNECTING) { + // each subchannel is responsible for its own backoff + return; + } else if (newState == IDLE) { + requestConnection(); + return; + } + } + + switch (newState) { + case IDLE: + // Shutdown when ready: connect from beginning when prompted + addressIndex.reset(); + updateBalancingState(IDLE, new RequestConnectionPicker(this));; + break; + case CONNECTING: + // It's safe to use RequestConnectionPicker here, so when coming from IDLE we could leave + // the current picker in-place. But ignoring the potential optimization is simpler. + updateBalancingState(CONNECTING, new Picker(PickResult.withNoResult())); + break; + case READY: + updateBalancingState(READY, new Picker(PickResult.withSubchannel(subchannel))); + shutdownRemaining(subchannel); + addressIndex.seekTo(getAddress(subchannel)); + break; + case TRANSIENT_FAILURE: + // If we are looking at current channel, request a connection if possible + if (addressIndex.isValid() + && subchannels.get(addressIndex.getCurrentAddress()).getSubchannel() == subchannel) { + addressIndex.increment(); + requestConnection(); + + // If no addresses remaining, go into TRANSIENT_FAILURE + if (!addressIndex.isValid()) { + helper.refreshNameResolution(); + updateBalancingState(TRANSIENT_FAILURE, + new Picker(PickResult.withError(stateInfo.getStatus()))); + } + } + break; + default: + throw new IllegalArgumentException("Unsupported state:" + newState); + } + } + + private void updateBalancingState(ConnectivityState state, SubchannelPicker picker) { + if (state != currentState || state == READY || state == TRANSIENT_FAILURE) { + currentState = state; + helper.updateBalancingState(state, picker); + } + } + + @Override + public void shutdown() { + for (SubchannelData subchannelData : subchannels.values()) { + subchannelData.getSubchannel().shutdown(); + } + subchannels.clear(); + } + + /** + * Shuts down remaining subchannels. Called when a subchannel becomes ready, which means + * that all other subchannels must be shutdown. + */ + private void shutdownRemaining(Subchannel activeSubchannel) { + for (SubchannelData subchannelData : subchannels.values()) { + if (!subchannelData.getSubchannel().equals(activeSubchannel)) { + subchannelData.getSubchannel().shutdown(); + } + } + subchannels.clear(); + subchannels.put(getAddress(activeSubchannel), new SubchannelData(activeSubchannel, READY)); + } + + /** + * Requests a connection to the next applicable address' subchannel, creating one if necessary + * If the current channel has already attempted a connection, we attempt a connection + * to the next address/subchannel in our list. + */ + @Override + public void requestConnection() { + if (subchannels.size() == 0) { + return; + } + if (addressIndex.isValid()) { + Subchannel subchannel = subchannels.containsKey(addressIndex.getCurrentAddress()) + ? subchannels.get(addressIndex.getCurrentAddress()).getSubchannel() + : createNewSubchannel(addressIndex.getCurrentAddress()); + + ConnectivityState subchannelState = + subchannels.get(addressIndex.getCurrentAddress()).getState(); + if (subchannelState == IDLE) { + subchannel.requestConnection(); + } else if (subchannelState == CONNECTING || subchannelState == TRANSIENT_FAILURE) { + addressIndex.increment(); + requestConnection(); + } + } + } + + private Subchannel createNewSubchannel(SocketAddress addr) { + final Subchannel subchannel = helper.createSubchannel( + CreateSubchannelArgs.newBuilder() + .setAddresses(Lists.newArrayList( + new EquivalentAddressGroup(addr))) + .build()); + subchannels.put(addr, new SubchannelData(subchannel, IDLE)); + subchannel.start(new SubchannelStateListener() { + @Override + public void onSubchannelState(ConnectivityStateInfo stateInfo) { + processSubchannelState(subchannel, stateInfo); + } + }); + return subchannel; + } + + private SocketAddress getAddress(Subchannel subchannel) { + return subchannel.getAddresses().getAddresses().get(0); + } + + @VisibleForTesting + ConnectivityState getCurrentState() { + return this.currentState; + } + + /** + * No-op picker which doesn't add any custom picking logic. It just passes already known result + * received in constructor. + */ + private static final class Picker extends SubchannelPicker { + private final PickResult result; + + Picker(PickResult result) { + this.result = checkNotNull(result, "result"); + } + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return result; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(Picker.class).add("result", result).toString(); + } + } + + /** + * Picker that requests connection during the first pick, and returns noResult. + */ + private final class RequestConnectionPicker extends SubchannelPicker { + private final PickFirstLeafLoadBalancer pickFirstLeafLoadBalancer; + private final AtomicBoolean connectionRequested = new AtomicBoolean(false); + + RequestConnectionPicker(PickFirstLeafLoadBalancer pickFirstLeafLoadBalancer) { + this.pickFirstLeafLoadBalancer = + checkNotNull(pickFirstLeafLoadBalancer, "pickFirstLeafLoadBalancer"); + } + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + if (connectionRequested.compareAndSet(false, true)) { + helper.getSynchronizationContext().execute(new Runnable() { + @Override + public void run() { + pickFirstLeafLoadBalancer.requestConnection(); + } + }); + } + return PickResult.withNoResult(); + } + } + + /** + * Index as in 'i', the pointer to an entry. Not a "search index." + */ + @VisibleForTesting + static final class Index { + private List addressGroups; + private int groupIndex; + private int addressIndex; + + public Index(List groups) { + this.addressGroups = groups; + } + + public boolean isValid() { + // addressIndex will never be invalid + return groupIndex < addressGroups.size(); + } + + public boolean isAtBeginning() { + return groupIndex == 0 && addressIndex == 0; + } + + public void increment() { + EquivalentAddressGroup group = addressGroups.get(groupIndex); + addressIndex++; + if (addressIndex >= group.getAddresses().size()) { + groupIndex++; + addressIndex = 0; + } + } + + public void reset() { + groupIndex = 0; + addressIndex = 0; + } + + public SocketAddress getCurrentAddress() { + return addressGroups.get(groupIndex).getAddresses().get(addressIndex); + } + + public Attributes getCurrentEagAttributes() { + return addressGroups.get(groupIndex).getAttributes(); + } + + public List getGroups() { + return addressGroups; + } + + /** + * Update to new groups, resetting the current index. + */ + public void updateGroups(List newGroups) { + addressGroups = newGroups; + reset(); + } + + /** + * Returns false if the needle was not found and the current index was left unchanged. + */ + public boolean seekTo(SocketAddress needle) { + for (int i = 0; i < addressGroups.size(); i++) { + EquivalentAddressGroup group = addressGroups.get(i); + int j = group.getAddresses().indexOf(needle); + if (j == -1) { + continue; + } + this.groupIndex = i; + this.addressIndex = j; + return true; + } + return false; + } + } + + private static final class SubchannelData { + private final Subchannel subchannel; + private ConnectivityState state; + + public SubchannelData(Subchannel subchannel, ConnectivityState state) { + this.subchannel = subchannel; + this.state = state; + } + + public Subchannel getSubchannel() { + return this.subchannel; + } + + public ConnectivityState getState() { + return this.state; + } + + private void updateState(ConnectivityState newState) { + this.state = newState; + } + } + + public static final class PickFirstLeafLoadBalancerConfig { + + @Nullable + public final Boolean shuffleAddressList; + + // For testing purposes only, not meant to be parsed from a real config. + @Nullable + final Long randomSeed; + + public PickFirstLeafLoadBalancerConfig(@Nullable Boolean shuffleAddressList) { + this(shuffleAddressList, null); + } + + PickFirstLeafLoadBalancerConfig(@Nullable Boolean shuffleAddressList, + @Nullable Long randomSeed) { + this.shuffleAddressList = shuffleAddressList; + this.randomSeed = randomSeed; + } + } +} diff --git a/core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java b/core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java new file mode 100644 index 00000000000..5437cbc9198 --- /dev/null +++ b/core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java @@ -0,0 +1,2131 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.IDLE; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; +import io.grpc.Attributes; +import io.grpc.ConnectivityState; +import io.grpc.ConnectivityStateInfo; +import io.grpc.EquivalentAddressGroup; +import io.grpc.LoadBalancer.CreateSubchannelArgs; +import io.grpc.LoadBalancer.Helper; +import io.grpc.LoadBalancer.PickResult; +import io.grpc.LoadBalancer.PickSubchannelArgs; +import io.grpc.LoadBalancer.ResolvedAddresses; +import io.grpc.LoadBalancer.Subchannel; +import io.grpc.LoadBalancer.SubchannelPicker; +import io.grpc.LoadBalancer.SubchannelStateListener; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; +import io.grpc.internal.PickFirstLeafLoadBalancer.PickFirstLeafLoadBalancerConfig; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + + +/** Unit test for {@link PickFirstLeafLoadBalancer}. */ +@RunWith(JUnit4.class) +public class PickFirstLeafLoadBalancerTest { + private PickFirstLeafLoadBalancer loadBalancer; + private final List servers = Lists.newArrayList(); + private static final Attributes.Key FOO = Attributes.Key.create("foo"); + + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + private final Attributes affinity = Attributes.newBuilder().set(FOO, "bar").build(); + @Rule + public final MockitoRule mocks = MockitoJUnit.rule(); + @Captor + private ArgumentCaptor pickerCaptor; + @Captor + private ArgumentCaptor connectivityStateCaptor; + @Captor + private ArgumentCaptor createArgsCaptor; + @Captor + private ArgumentCaptor stateListenerCaptor; + @Mock + private Helper mockHelper; + @Mock + private FakeSubchannel mockSubchannel1; + @Mock + private FakeSubchannel mockSubchannel2; + @Mock + private FakeSubchannel mockSubchannel3; + @Mock + private FakeSubchannel mockSubchannel4; + @Mock // This LoadBalancer doesn't use any of the arg fields, as verified in tearDown(). + private PickSubchannelArgs mockArgs; + + @Before + public void setUp() { + for (int i = 1; i < 5; i++) { + SocketAddress addr = new FakeSocketAddress("server" + i); + servers.add(new EquivalentAddressGroup(addr)); + } + mockSubchannel1 = new FakeSubchannel(Lists.newArrayList( + new EquivalentAddressGroup(new FakeSocketAddress("fake"))), null); + mockSubchannel1 = mock(FakeSubchannel.class); + mockSubchannel2 = mock(FakeSubchannel.class); + mockSubchannel3 = mock(FakeSubchannel.class); + mockSubchannel4 = mock(FakeSubchannel.class); + when(mockHelper.createSubchannel(any(CreateSubchannelArgs.class))) + .thenReturn(mockSubchannel1, mockSubchannel2, mockSubchannel3, mockSubchannel4); + + when(mockSubchannel1.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + when(mockSubchannel2.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(1))); + when(mockSubchannel3.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(2))); + when(mockSubchannel4.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(3))); + + when(mockHelper.getSynchronizationContext()).thenReturn(syncContext); + loadBalancer = new PickFirstLeafLoadBalancer(mockHelper); + } + + @After + public void tearDown() throws Exception { + verifyNoMoreInteractions(mockArgs); + } + + @Test + public void pickAfterResolved() throws Exception { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolved_shuffle() throws Exception { + servers.remove(3); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity) + .setLoadBalancingPolicyConfig(new PickFirstLeafLoadBalancerConfig(true, 123L)).build()); + + verify(mockHelper, times(3)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + // We should still see the same set of addresses. + // Because we use a fixed seed, the addresses should always be shuffled in this order. + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolved_noShuffle() throws Exception { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity) + .setLoadBalancingPolicyConfig(new PickFirstLeafLoadBalancerConfig(false)).build()); + + verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void requestConnectionPicker() throws Exception { + // Set up + assertEquals(IDLE, loadBalancer.getCurrentState()); + List newServers = Lists.newArrayList(servers.get(0), servers.get(1), + servers.get(2)); + + // Accepting resolved addresses + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, mockSubchannel3); + + // We initialize and start all subchannels + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + + // We start connection attempt to the first address in the list + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // If we send the first subchannel into idle ... + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + + SubchannelPicker picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() requests a connection, gives the same result when called twice. + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once for a total of two connection requests. + inOrder.verify(mockSubchannel1).requestConnection(); + verify(mockSubchannel1, times(2)).requestConnection(); + } + + @Test + public void refreshNameResolutionAfterSubchannelConnectionBroken() { + List newServers = Lists.newArrayList(servers.get(0)); + when(mockSubchannel1.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + + // accept resolved addresses + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + InOrder inOrder = inOrder(mockHelper, mockSubchannel1); + verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertNull(pickerCaptor.getValue().pickSubchannel(mockArgs).getSubchannel()); + inOrder.verify(mockSubchannel1).requestConnection(); + + Status error = Status.UNAUTHENTICATED.withDescription("permission denied"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + assertEquals(mockSubchannel1, pickerCaptor.getValue().pickSubchannel(mockArgs).getSubchannel()); + + // Simulate receiving go-away so the subchannel transit to IDLE. + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), any(SubchannelPicker.class)); + } + + @Test + public void pickAfterResolvedAndUnchanged() throws Exception { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + verify(mockSubchannel1).requestConnection(); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockSubchannel1).requestConnection(); + + verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + assertThat(createArgsCaptor.getValue()).isNotNull(); + verify(mockHelper) + .updateBalancingState(isA(ConnectivityState.class), isA(SubchannelPicker.class)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolvedAndChanged() throws Exception { + SocketAddress socketAddr1 = new FakeSocketAddress("oldserver"); + List oldServers = + Lists.newArrayList(new EquivalentAddressGroup(socketAddr1)); + + SocketAddress socketAddr2 = new FakeSocketAddress("newserver"); + List newServers = + Lists.newArrayList(new EquivalentAddressGroup(socketAddr2)); + + InOrder inOrder = inOrder(mockHelper, mockSubchannel1); + + // accept resolved addresses + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1).start(any(SubchannelStateListener.class)); + + // start connection attempt to first address + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + assertNull(pickerCaptor.getValue().pickSubchannel(mockArgs).getSubchannel()); + + // updating the subchannel addresses is unnecessary, but doesn't hurt anything + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + verify(mockSubchannel1).shutdown(); + + verifyNoMoreInteractions(mockSubchannel1); + verify(mockSubchannel2).start(any(SubchannelStateListener.class)); + } + + @Test + public void pickAfterStateChangeAfterResolution() throws Exception { + InOrder inOrder = inOrder(mockHelper); + + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + inOrder.verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + reset(mockHelper); + when(mockHelper.getSynchronizationContext()).thenReturn(syncContext); + + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + + // subchannel reports connecting when pick subchannel is called + assertEquals(Status.OK, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + + Status error = Status.UNAVAILABLE.withDescription("boom!"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + stateListener4.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(mockSubchannel1, picker.pickSubchannel(mockArgs).getSubchannel()); + + verify(mockHelper, atLeast(0)).getSynchronizationContext(); // Don't care + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolutionAfterTransientValue() throws Exception { + InOrder inOrder = inOrder(mockHelper); + List newServers = Lists.newArrayList(servers.get(0)); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + reset(mockHelper); + when(mockHelper.getSynchronizationContext()).thenReturn(syncContext); + + // An error has happened. + Status error = Status.UNAVAILABLE.withDescription("boom!"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + inOrder.verify(mockHelper).refreshNameResolution(); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + + // Transition from TRANSIENT_ERROR to CONNECTING should also be ignored. + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + verifyNoMoreInteractions(mockHelper); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + } + + @Test + public void nameResolutionError() throws Exception { + Status error = Status.NOT_FOUND.withDescription("nameResolutionError"); + loadBalancer.handleNameResolutionError(error); + verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + PickResult pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertNull(pickResult.getSubchannel()); + assertEquals(error, pickResult.getStatus()); + verify(mockSubchannel1, never()).requestConnection(); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void nameResolutionError_emptyAddressList() throws Exception { + servers.clear(); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockHelper).updateBalancingState(connectivityStateCaptor.capture(), + pickerCaptor.capture()); + PickResult pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertThat(pickResult.getSubchannel()).isNull(); + assertThat(pickResult.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(pickResult.getStatus().getDescription()).contains("returned no usable address"); + verify(mockSubchannel1, never()).requestConnection(); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void nameResolutionSuccessAfterError() throws Exception { + InOrder inOrder = inOrder(mockHelper); + + loadBalancer.handleNameResolutionError(Status.NOT_FOUND.withDescription("nameResolutionError")); + inOrder.verify(mockHelper) + .updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class)); + verify(mockSubchannel1, never()).requestConnection(); + + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + inOrder.verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(0).getAttributes()).isEqualTo(Attributes.EMPTY); + assertThat(argsList.get(1).getAttributes()).isEqualTo(Attributes.EMPTY); + assertThat(argsList.get(2).getAttributes()).isEqualTo(Attributes.EMPTY); + assertThat(argsList.get(3).getAttributes()).isEqualTo(Attributes.EMPTY); + + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + assertNull(pickerCaptor.getValue().pickSubchannel(mockArgs) + .getSubchannel()); + + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void nameResolutionErrorWithStateChanges() throws Exception { + List newServers = Lists.newArrayList(servers.get(0)); + InOrder inOrder = inOrder(mockHelper); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState( + eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + Status error = Status.NOT_FOUND.withDescription("nameResolutionError"); + loadBalancer.handleNameResolutionError(error); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + PickResult pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertNull(pickResult.getSubchannel()); + assertEquals(error, pickResult.getStatus()); + + Status error2 = Status.NOT_FOUND.withDescription("nameResolutionError2"); + loadBalancer.handleNameResolutionError(error2); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + + pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertNull(pickResult.getSubchannel()); + assertEquals(error2, pickResult.getStatus()); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void requestConnection() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // calling requestConnection() starts next subchannel + loadBalancer.requestConnection(); + inOrder.verify(mockSubchannel2).requestConnection(); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // calling requestConnection is now a no-op + loadBalancer.requestConnection(); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void updateAddresses_emptyEagList_returns_false() { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + assertFalse(loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Arrays.asList()).setAttributes(affinity).build())); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + } + + @Test + public void updateAddresses_eagListWithNull_returns_false() { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + List eags = Arrays.asList((EquivalentAddressGroup) null); + assertFalse(loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(eags).setAttributes(affinity).build())); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + } + + @Test + public void updateAddresses_disjoint_idle() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Going into IDLE state + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + + // Creating second set of disjoint endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // We create new channels, remove old ones, and keep intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + verify(mockSubchannel1).shutdown(); + verify(mockSubchannel2).shutdown(); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel3).requestConnection(); + + // Ready subchannel 3 + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + + // Picking a subchannel returns subchannel 3 + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_disjoint_connecting() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Old subchannels should shut down (in no particular order) and request a connection + verify(mockSubchannel1).shutdown(); + verify(mockSubchannel2).shutdown(); + inOrder.verify(mockSubchannel3).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // If old subchannel becomes ready, the state should not be affected + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Fail connection attempt to third address + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Verify starting connection attempt to fourth address + inOrder.verify(mockSubchannel4).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Succeed connection attempt to fourth address + stateListener4.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_disjoint_ready_twice() { + when(mockHelper.createSubchannel(any(CreateSubchannelArgs.class))) + .thenReturn(mockSubchannel1, mockSubchannel2, mockSubchannel3, + mockSubchannel4, mockSubchannel1, mockSubchannel2); + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection shuts down other subchannel + inOrder.verify(mockSubchannel2).shutdown(); + + // Verify that picker returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).shutdown(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel3).requestConnection(); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withNoResult(), picker.pickSubchannel(mockArgs)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Ready subchannel 3 + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection shuts down other subchannel + inOrder.verify(mockSubchannel4).shutdown(); + + // Verify that pickSubchannel() returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Creating third set of endpoints/addresses + List newestServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Second address update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newestServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel3).shutdown(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(PickResult.withNoResult(), pickerCaptor.getValue().pickSubchannel(mockArgs)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener4.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Connection attempt to address 1 is unsuccessful + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting connection attempt to address 2 + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Connection attempt to address 2 is successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection shuts down other subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel1).shutdown(); + + // Verify that picker still returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_disjoint_transient_failure() { + // Starting first connection attempt + when(mockHelper.createSubchannel(any(CreateSubchannelArgs.class))) + .thenReturn(mockSubchannel1, mockSubchannel2, mockSubchannel3, + mockSubchannel4, mockSubchannel1, mockSubchannel2); + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List addrs = Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); // sticky transient failure + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // subchannel 3 still attempts a connection even though we stay in transient failure + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + verify(mockSubchannel1).shutdown(); + verify(mockSubchannel2).shutdown(); + inOrder.verify(mockSubchannel3).requestConnection(); + + // Obselete subchannels should not affect us + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Third subchannel connection attempt is unsuccessful + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + inOrder.verify(mockSubchannel4).requestConnection(); + + // Fourth subchannel connection attempt is successful + stateListener4.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Picking a subchannel returns subchannel 3 + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel3).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_idle() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // First connection attempt is successful + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection attempt shuts down other subchannels + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + + // Verify that picker returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Going into IDLE state, nothing should happen unless requested + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + verifyNoMoreInteractions(mockHelper); + + // Creating second set of intersecting endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // We create new channels and remove old ones, keeping intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel1).requestConnection(); + + // internal subchannel calls back and reports connecting + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(PickResult.withNoResult(), pickerCaptor.getValue().pickSubchannel(mockArgs)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Ready subchannel 1 + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + + // Picking a subchannel returns subchannel 1 + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_connecting() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(2)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // callback from internal subchannel + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // Creating second set of endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Don't unnecessarily create new subchannels and keep intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + verifyNoMoreInteractions(mockHelper); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_ready() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Creating second set of endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(2)); + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // The state is still READY after update since we had an intersecting subchannel that was READY. + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker still returns correct subchannel + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_transient_failure() { + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); // sticky transient failure + + // Creating second set of endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(2)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // subchannel 3 still attempts a connection even though we stay in transient failure + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + inOrder.verify(mockSubchannel3).requestConnection(); + + // no other connections should be requested by LB, we should come out of backoff to request + verifyNoMoreInteractions(mockSubchannel3); + + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Second subchannel connection attempt is now successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + + // Picking a subchannel returns subchannel 3 + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel3).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_enter_transient_failure() { + // after an address update occurs, verify that the client properly tries all + // addresses and only then enters transient failure. + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // callback from internal subchannel + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(0), servers.get(2)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // We create new channels and remove old ones, keeping intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is unsuccessful + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Subchannel 3 attempt starts but fails + inOrder.verify(mockSubchannel3).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + } + + @Test + public void updateAddresses_identical_idle() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Going into IDLE state + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(3)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + + // First connection attempt is successful + assertEquals(IDLE, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_identical_connecting() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(2)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + + // First connection attempt is successful + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_identical_ready() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel2).shutdown(); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(2)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker hasn't changed via checking mock helper's interactions + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void updateAddresses_identical_transient_failure() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is unsuccessful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second connection attempt is unsuccessful + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(2)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // No new connections are requested, subchannels responsible for completing their own backoffs + verifyNoMoreInteractions(mockHelper); + + // First connection attempt is successful + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void twoAddressesSeriallyConnect() { + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, mockSubchannel3); + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second connection attempt is successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void multiple_backoffs() { + // This test case mimics a backoff without implementing one + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Starting first connection attempt + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for second address + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Failing first connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for second address + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Connection attempt to second address is now successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + when(mockSubchannel2.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + inOrder.verify(mockSubchannel1).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // If first subchannel is ready before it completes shutdown, we still choose subchannel 2 + // This can be verified by checking the mock helper. + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void backoff_faster_than_serial_connection() { + // this tests the case where a subchannel completes its backoff and readies a connection + // before the other addresses have a chance to complete their connection attempts + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = Lists.newArrayList(servers.get(0), + servers.get(1)); + + // Accepting resolved addresses starts all subchannels + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Connection attempt to first address is now successful + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void success_from_transient_failure() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accepting resolved addresses starts all subchannels + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); // sticky transient failure + + // Failing connection attempt to first address + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for second address + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Connection attempt to second address is now successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // If first address is successful, nothing happens. Verify by checking mock helper + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void lastAddressFailingNotTransientFailure() { + // This tests the case where after an address update, the last address escapes a backoff + // and reports transient failure before all addresses have failed connection + // attempts, in which case we should not report TRANSIENT_FAILURE. + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is unsuccessful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second connection attempt is connecting + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Accept same resolved addresses to update + List newServers = Lists.newArrayList(servers.get(2), servers.get(1)); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel1).shutdown(); + inOrder.verify(mockSubchannel3).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second address connection attempt is unsuccessful, but should not go into transient failure + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Third address connection attempt is unsuccessful, now we enter transient failure + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Obselete subchannels have no impact + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Second subchannel is successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel3).shutdown(); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void recreate_shutdown_subchannel() { + // Take the case where the latter subchannel is readied. If we then go to an IDLE state and + // re-request a connection, we should start and create a new subchannel for the first + // address in our list. + + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = + Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Successful second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockSubchannel1).shutdown(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Go to IDLE + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + SubchannelPicker picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() requests a connection, gives the same result when called twice. + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel3).requestConnection(); + when(mockSubchannel3.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // first subchannel connection attempt fails + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // second subchannel connection attempt + inOrder.verify(mockSubchannel2).requestConnection(); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel3).shutdown(); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void ready_then_transient_failure_again() { + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = + Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Successful second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockSubchannel1).shutdown(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Go to IDLE + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + SubchannelPicker picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() requests a connection, gives the same result when called twice. + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel3).requestConnection(); + when(mockSubchannel3.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // first subchannel connection attempt fails + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // second subchannel connection attempt + inOrder.verify(mockSubchannel2).requestConnection(); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel3).shutdown(); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + + @Test + public void index_looping() { + Attributes.Key key = Attributes.Key.create("some-key"); + Attributes attr1 = Attributes.newBuilder().set(key, "1").build(); + Attributes attr2 = Attributes.newBuilder().set(key, "2").build(); + Attributes attr3 = Attributes.newBuilder().set(key, "3").build(); + SocketAddress addr1 = new FakeSocketAddress("addr1"); + SocketAddress addr2 = new FakeSocketAddress("addr2"); + SocketAddress addr3 = new FakeSocketAddress("addr3"); + SocketAddress addr4 = new FakeSocketAddress("addr4"); + SocketAddress addr5 = new FakeSocketAddress("addr5"); + PickFirstLeafLoadBalancer.Index index = new PickFirstLeafLoadBalancer.Index(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1, addr2), attr1), + new EquivalentAddressGroup(Arrays.asList(addr3), attr2), + new EquivalentAddressGroup(Arrays.asList(addr4, addr5), attr3))); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + assertThat(index.isAtBeginning()).isTrue(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr2); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr3); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr2); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr4); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr3); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr5); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr3); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isFalse(); + + index.reset(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + assertThat(index.isAtBeginning()).isTrue(); + assertThat(index.isValid()).isTrue(); + + // We want to make sure both groupIndex and addressIndex are reset + index.increment(); + index.increment(); + index.increment(); + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr5); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr3); + index.reset(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + } + + @Test + public void index_updateGroups_resets() { + SocketAddress addr1 = new FakeSocketAddress("addr1"); + SocketAddress addr2 = new FakeSocketAddress("addr2"); + SocketAddress addr3 = new FakeSocketAddress("addr3"); + PickFirstLeafLoadBalancer.Index index = new PickFirstLeafLoadBalancer.Index(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1)), + new EquivalentAddressGroup(Arrays.asList(addr2, addr3)))); + index.increment(); + index.increment(); + // We want to make sure both groupIndex and addressIndex are reset + index.updateGroups(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1)), + new EquivalentAddressGroup(Arrays.asList(addr2, addr3)))); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + } + + @Test + public void index_seekTo() { + SocketAddress addr1 = new FakeSocketAddress("addr1"); + SocketAddress addr2 = new FakeSocketAddress("addr2"); + SocketAddress addr3 = new FakeSocketAddress("addr3"); + PickFirstLeafLoadBalancer.Index index = new PickFirstLeafLoadBalancer.Index(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1, addr2)), + new EquivalentAddressGroup(Arrays.asList(addr3)))); + assertThat(index.seekTo(addr3)).isTrue(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr3); + assertThat(index.seekTo(addr1)).isTrue(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.seekTo(addr2)).isTrue(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr2); + index.seekTo(new FakeSocketAddress("addr4")); + // Failed seekTo doesn't change the index + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr2); + } + + private static class FakeSocketAddress extends SocketAddress { + final String name; + + FakeSocketAddress(String name) { + this.name = name; + } + + @Override + public String toString() { + return "FakeSocketAddress-" + name; + } + + } + + private static class FakeSubchannel extends Subchannel { + private final Attributes attributes; + private List eags; + private SubchannelStateListener listener; + + public FakeSubchannel(List eags, Attributes attributes) { + this.eags = Collections.unmodifiableList(eags); + this.attributes = attributes; + } + + @Override + public List getAllAddresses() { + return eags; + } + + @Override + public Attributes getAttributes() { + return attributes; + } + + @Override + public void start(SubchannelStateListener listener) { + this.listener = checkNotNull(listener, "listener"); + } + + @Override + public void updateAddresses(List addrs) { + this.eags = Collections.unmodifiableList(addrs); + } + + @Override + public void shutdown() { + } + + @Override + public void requestConnection() { + listener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + } + } +} \ No newline at end of file From 849186ac3562700568d46ced0bd7327af7e333da Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 14 Aug 2023 17:00:29 -0700 Subject: [PATCH 10/14] examples: Add pre-serialized-message example (#10112) This came out of the question #9707, and could be useful to others. --- examples/README.md | 2 + examples/build.gradle | 2 + .../preserialized/ByteArrayMarshaller.java | 43 +++++ .../preserialized/PreSerializedClient.java | 108 ++++++++++++ .../preserialized/PreSerializedServer.java | 164 ++++++++++++++++++ 5 files changed, 319 insertions(+) create mode 100644 examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java create mode 100644 examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java create mode 100644 examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java diff --git a/examples/README.md b/examples/README.md index ae849fa7d6d..9664942b5fa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -205,6 +205,8 @@ $ bazel-bin/hello-world-client - [JWT-based Authentication](example-jwt-auth) +- [Pre-serialized messages](src/main/java/io/grpc/examples/preserialized) + ## Unit test examples Examples for unit testing gRPC clients and servers are located in [examples/src/test](src/test). diff --git a/examples/build.gradle b/examples/build.gradle index fb799b42eda..7322ae29082 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -110,6 +110,8 @@ createStartScripts('io.grpc.examples.multiplex.MultiplexingServer') createStartScripts('io.grpc.examples.multiplex.SharingClient') createStartScripts('io.grpc.examples.nameresolve.NameResolveClient') createStartScripts('io.grpc.examples.nameresolve.NameResolveServer') +createStartScripts('io.grpc.examples.preserialized.PreSerializedClient') +createStartScripts('io.grpc.examples.preserialized.PreSerializedServer') createStartScripts('io.grpc.examples.retrying.RetryingHelloWorldClient') createStartScripts('io.grpc.examples.retrying.RetryingHelloWorldServer') createStartScripts('io.grpc.examples.routeguide.RouteGuideClient') diff --git a/examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java b/examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java new file mode 100644 index 00000000000..c6f099280f9 --- /dev/null +++ b/examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.examples.preserialized; + +import com.google.common.io.ByteStreams; +import io.grpc.MethodDescriptor; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * A marshaller that produces a byte[] instead of decoding into typical POJOs. It can be used for + * any message type. + */ +final class ByteArrayMarshaller implements MethodDescriptor.Marshaller { + @Override + public byte[] parse(InputStream stream) { + try { + return ByteStreams.toByteArray(stream); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public InputStream stream(byte[] b) { + return new ByteArrayInputStream(b); + } +} diff --git a/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java new file mode 100644 index 00000000000..511e8a177f8 --- /dev/null +++ b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java @@ -0,0 +1,108 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.examples.preserialized; + +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.Grpc; +import io.grpc.InsecureChannelCredentials; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; +import io.grpc.examples.helloworld.GreeterGrpc; +import io.grpc.examples.helloworld.HelloReply; +import io.grpc.examples.helloworld.HelloRequest; +import io.grpc.stub.ClientCalls; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A client that requests a greeting from a hello-world server, but using a pre-serialized request. + * This is a performance optimization that can be useful if you read the request from on-disk or a + * database where it is already serialized, or if you need to send the same complicated message to + * many servers. The same approach can avoid deserializing responses, to be stored in a database. + * This adjustment is client-side only; the server is unable to detect the difference, so this + * client is fully-compatible with the normal {@link HelloWorldServer}. + */ +public class PreSerializedClient { + private static final Logger logger = Logger.getLogger(PreSerializedClient.class.getName()); + + /** + * Modified sayHello() descriptor with bytes as the request, instead of HelloRequest. By adjusting + * toBuilder() you can choose which of the request and response are bytes. + */ + private static final MethodDescriptor SAY_HELLO + = GreeterGrpc.getSayHelloMethod() + .toBuilder(new ByteArrayMarshaller(), GreeterGrpc.getSayHelloMethod().getResponseMarshaller()) + .build(); + + private final Channel channel; + + /** Construct client for accessing hello-world server using the existing channel. */ + public PreSerializedClient(Channel channel) { + this.channel = channel; + } + + /** Say hello to server. */ + public void greet(String name) { + logger.info("Will try to greet " + name + " ..."); + byte[] request = HelloRequest.newBuilder().setName(name).build().toByteArray(); + HelloReply response; + try { + // Stubs use ClientCalls to send RPCs. Since the generated stub won't have byte[] in its + // method signature, this uses ClientCalls directly. It isn't as convenient, but it behaves + // the same as a normal stub. + response = ClientCalls.blockingUnaryCall(channel, SAY_HELLO, CallOptions.DEFAULT, request); + } catch (StatusRuntimeException e) { + logger.log(Level.WARNING, "RPC failed: {0}", e.getStatus()); + return; + } + logger.info("Greeting: " + response.getMessage()); + } + + /** + * Greet server. If provided, the first element of {@code args} is the name to use in the + * greeting. The second argument is the target server. + */ + public static void main(String[] args) throws Exception { + String user = "world"; + String target = "localhost:50051"; + if (args.length > 0) { + if ("--help".equals(args[0])) { + System.err.println("Usage: [name [target]]"); + System.err.println(""); + System.err.println(" name The name you wish to be greeted by. Defaults to " + user); + System.err.println(" target The server to connect to. Defaults to " + target); + System.exit(1); + } + user = args[0]; + } + if (args.length > 1) { + target = args[1]; + } + + ManagedChannel channel = Grpc.newChannelBuilder(target, InsecureChannelCredentials.create()) + .build(); + try { + PreSerializedClient client = new PreSerializedClient(channel); + client.greet(user); + } finally { + channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + } + } +} diff --git a/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java new file mode 100644 index 00000000000..51beca57386 --- /dev/null +++ b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java @@ -0,0 +1,164 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.examples.preserialized; + +import io.grpc.BindableService; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; +import io.grpc.MethodDescriptor; +import io.grpc.Server; +import io.grpc.ServerCallHandler; +import io.grpc.ServerMethodDefinition; +import io.grpc.ServerServiceDefinition; +import io.grpc.ServiceDescriptor; +import io.grpc.examples.helloworld.GreeterGrpc; +import io.grpc.examples.helloworld.HelloReply; +import io.grpc.examples.helloworld.HelloRequest; +import io.grpc.stub.ServerCalls; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Server that provides a {@code Greeter} service, but that uses a pre-serialized response. This is + * a performance optimization that can be useful if you read the response from on-disk or a database + * where it is already serialized, or if you need to send the same complicated message to many + * clients. The same approach can avoid deserializing requests, to be stored in a database. This + * adjustment is server-side only; the client is unable to detect the differences, so this server is + * fully-compatible with the normal {@link HelloWorldClient}. + */ +public class PreSerializedServer { + private static final Logger logger = Logger.getLogger(PreSerializedServer.class.getName()); + + private Server server; + + private void start() throws IOException { + int port = 50051; + server = Grpc.newServerBuilderForPort(port, InsecureServerCredentials.create()) + .addService(new GreeterImpl()) + .build() + .start(); + logger.info("Server started, listening on " + port); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + // Use stderr here since the logger may have been reset by its JVM shutdown hook. + System.err.println("*** shutting down gRPC server since JVM is shutting down"); + try { + PreSerializedServer.this.stop(); + } catch (InterruptedException e) { + e.printStackTrace(System.err); + } + System.err.println("*** server shut down"); + } + }); + } + + private void stop() throws InterruptedException { + if (server != null) { + server.shutdown().awaitTermination(30, TimeUnit.SECONDS); + } + } + + /** + * Await termination on the main thread since the grpc library uses daemon threads. + */ + private void blockUntilShutdown() throws InterruptedException { + if (server != null) { + server.awaitTermination(); + } + } + + /** + * Main launches the server from the command line. + */ + public static void main(String[] args) throws IOException, InterruptedException { + final PreSerializedServer server = new PreSerializedServer(); + server.start(); + server.blockUntilShutdown(); + } + + static class GreeterImpl implements GreeterGrpc.AsyncService, BindableService { + + public void byteSayHello(HelloRequest req, StreamObserver responseObserver) { + HelloReply reply = HelloReply.newBuilder().setMessage("Hello " + req.getName()).build(); + responseObserver.onNext(reply.toByteArray()); + responseObserver.onCompleted(); + } + + @Override + public ServerServiceDefinition bindService() { + MethodDescriptor sayHello = GreeterGrpc.getSayHelloMethod(); + // Modifying the method descriptor to use bytes as the response, instead of HelloReply. By + // adjusting toBuilder() you can choose which of the request and response are bytes. + MethodDescriptor byteSayHello = sayHello + .toBuilder(sayHello.getRequestMarshaller(), new ByteArrayMarshaller()) + .build(); + // GreeterGrpc.bindService() will bind every service method, including sayHello(). (Although + // Greeter only has one method, this approach would work for any service.) AsyncService + // provides a default implementation of sayHello() that returns UNIMPLEMENTED, and that + // implementation will be used by bindService(). replaceMethod() will rewrite that method to + // use our byte-based method instead. + // + // The generated bindService() uses ServerCalls to make RPC handlers. Since the generated + // bindService() won't expect byte[] in the AsyncService, this uses ServerCalls directly. It + // isn't as convenient, but it behaves the same as a normal RPC handler. + return replaceMethod( + GreeterGrpc.bindService(this), + byteSayHello, + ServerCalls.asyncUnaryCall(this::byteSayHello)); + } + + /** Rewrites the ServerServiceDefinition replacing one method's definition. */ + private static ServerServiceDefinition replaceMethod( + ServerServiceDefinition def, + MethodDescriptor newDesc, + ServerCallHandler newHandler) { + // There are two data structures involved. The first is the "descriptor" which describes the + // service and methods as a schema. This is the same on client and server. The second is the + // "definition" which includes the handlers to execute methods. This is specific to the server + // and is generated by "bind." This adjusts both the descriptor and definition. + + // Descriptor + ServiceDescriptor desc = def.getServiceDescriptor(); + ServiceDescriptor.Builder descBuilder = ServiceDescriptor.newBuilder(desc.getName()) + .setSchemaDescriptor(desc.getSchemaDescriptor()) + .addMethod(newDesc); // Add the modified method + // Copy methods other than the modified one + for (MethodDescriptor md : desc.getMethods()) { + if (newDesc.getFullMethodName().equals(md.getFullMethodName())) { + continue; + } + descBuilder.addMethod(md); + } + + // Definition + ServerServiceDefinition.Builder defBuilder = + ServerServiceDefinition.builder(descBuilder.build()) + .addMethod(newDesc, newHandler); // Add the modified method + // Copy methods other than the modified one + for (ServerMethodDefinition smd : def.getMethods()) { + if (newDesc.getFullMethodName().equals(smd.getMethodDescriptor().getFullMethodName())) { + continue; + } + defBuilder.addMethod(smd); + } + return defBuilder.build(); + } + } +} From 9585bc948ab02bb42043a2085ac65aae27c20ab5 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Fri, 11 Aug 2023 20:46:04 -0700 Subject: [PATCH 11/14] build.gradle: Modernize japicmp confiuration Even though we don't really use japicmp, the configuration was resolving artifacts even when the task was not executed. After some clean up, the configuration looks more ordinary. --- build.gradle | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index e0a905f2452..4d9f3a62df0 100644 --- a/build.gradle +++ b/build.gradle @@ -417,31 +417,25 @@ subprojects { def baselineGrpcVersion = '1.6.1' // Get the baseline version's jar for this subproject - File baselineArtifact = null - // Use a detached configuration, otherwise the current version's jar will take precedence - // over the baseline jar. + configurations { + baselineArtifact + } // A necessary hack, the intuitive thing does NOT work: // https://discuss.gradle.org/t/is-the-default-configuration-leaking-into-independent-configurations/2088/6 def oldGroup = project.group try { project.group = 'virtual_group_for_japicmp' - String depModule = "io.grpc:${project.name}:${baselineGrpcVersion}@jar" - String depJar = "${project.name}-${baselineGrpcVersion}.jar" - Configuration configuration = configurations.detachedConfiguration( - dependencies.create(depModule) - ) - baselineArtifact = files(configuration.files).filter { - it.name.equals(depJar) - }.singleFile + dependencies { + baselineArtifact "io.grpc:${project.name}:${baselineGrpcVersion}@jar" + } } finally { project.group = oldGroup } // Add a japicmp task that compares the current .jar with baseline .jar tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) { - dependsOn jar - oldClasspath.from baselineArtifact - newClasspath.from jar.archiveFile + oldClasspath.from configurations.baselineArtifact + newClasspath.from tasks.named("jar") onlyBinaryIncompatibleModified = false // Be quiet about things that did not change onlyModified = true @@ -454,12 +448,10 @@ subprojects { // Also break on source incompatible changes, not just binary. // Eg adding abstract method to public class. - // TODO(zpencer): enable after japicmp-gradle-plugin/pull/14 - // breakOnSourceIncompatibility = true + failOnSourceIncompatibility = true // Ignore any classes or methods marked @ExperimentalApi - // TODO(zpencer): enable after japicmp-gradle-plugin/pull/15 - // annotationExcludes = ['@io.grpc.ExperimentalApi'] + annotationExcludes = ['@io.grpc.ExperimentalApi'] } } } From 5850de2f5f18c8584bbe842a1d31c8262370400f Mon Sep 17 00:00:00 2001 From: Mohan Li <67390330+mohanli-ml@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:48:24 -0700 Subject: [PATCH 12/14] pick_first: de-experiment pick first (#10475) --- xds/src/main/java/io/grpc/xds/XdsResourceType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsResourceType.java b/xds/src/main/java/io/grpc/xds/XdsResourceType.java index 86fbf3fd6b9..1bec72d492c 100644 --- a/xds/src/main/java/io/grpc/xds/XdsResourceType.java +++ b/xds/src/main/java/io/grpc/xds/XdsResourceType.java @@ -58,7 +58,7 @@ abstract class XdsResourceType { static boolean enableWrr = getFlag("GRPC_EXPERIMENTAL_XDS_WRR_LB", true); @VisibleForTesting - static boolean enablePickFirst = getFlag("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", false); + static boolean enablePickFirst = getFlag("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", true); static final String TYPE_URL_CLUSTER_CONFIG = "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig"; From b5d7f1394df732c91f92c5df692b260ea15ae9ce Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 8 Jun 2023 08:48:07 -0700 Subject: [PATCH 13/14] xds: Fix import scripts deleting the wrong files, use of trap, and full git clone The scripts used `git rev-parse --show-toplevel` so it appeared they could be used from any directory. But references to "GIT_BASE_DIR" weren't absolute, so it did matter the starting directory. And it mattered in a big way for xds/import.sh as if you ran it from the grpc-java directory it would delete the xds directory in grpc-java, not third_party. The trap that deleted the GIT_BASE_DIR was very broken. In addition to potentially deleting the wrong directory, it was unnecessary because that directory was in tmpdir. But you can only have one trap per signal, so this unnecessary trap disabled the trap that deleted tmpdir. The script needed a full clone because it needed to check out a specific commit. To work with --depth 1 you have to use some convoluted syntax. But just downloading a tar.gz is easy and seems should work fine on Mac. protoc-gen-validate/import.sh didn't have the trap problem, but seemed to have drifted from the other scritps. All the scripts were synced to match. --- xds/third_party/envoy/import.sh | 23 +++++-------- xds/third_party/googleapis/import.sh | 21 ++++-------- xds/third_party/protoc-gen-validate/import.sh | 32 +++++++++---------- xds/third_party/xds/import.sh | 23 +++++-------- 4 files changed, 38 insertions(+), 61 deletions(-) diff --git a/xds/third_party/envoy/import.sh b/xds/third_party/envoy/import.sh index c4b5a8516f3..b85b0667800 100755 --- a/xds/third_party/envoy/import.sh +++ b/xds/third_party/envoy/import.sh @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update VERSION then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=main # import VERSION from the google internal copybara_version.txt for Envoy VERSION=0478eba2a495027bf6ac8e787c42e2f5b9eb553b -GIT_REPO="https://github.com/envoyproxy/envoy.git" -GIT_BASE_DIR=envoy -SOURCE_PROTO_BASE_DIR=envoy/api +DOWNLOAD_URL="https://github.com/envoyproxy/envoy/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="envoy-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}/api" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( @@ -181,19 +180,13 @@ envoy/type/v3/semantic_version.proto pushd `git rev-parse --show-toplevel`/xds/third_party/envoy -# clone the envoy github repo in a tmp directory +# put the repo in a tmp directory tmpdir="$(mktemp -d)" trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -pushd "${tmpdir}" -git clone -b $BRANCH $GIT_REPO -trap "rm -rf $GIT_BASE_DIR" EXIT -cd "$GIT_BASE_DIR" -git checkout $VERSION -popd - -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE -cp -p "${tmpdir}/${GIT_BASE_DIR}/NOTICE" NOTICE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/NOTICE" NOTICE rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" diff --git a/xds/third_party/googleapis/import.sh b/xds/third_party/googleapis/import.sh index e83536564e1..d51893e23d4 100755 --- a/xds/third_party/googleapis/import.sh +++ b/xds/third_party/googleapis/import.sh @@ -13,14 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update VERSION then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=master VERSION=ca1372c6d7bcb199638ebfdb40d2b2660bab7b88 -GIT_REPO="https://github.com/googleapis/googleapis.git" -GIT_BASE_DIR=googleapis -SOURCE_PROTO_BASE_DIR=googleapis +DOWNLOAD_URL="https://github.com/googleapis/googleapis/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="googleapis-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( @@ -30,18 +29,12 @@ google/api/expr/v1alpha1/syntax.proto pushd `git rev-parse --show-toplevel`/xds/third_party/googleapis -# clone the googleapis github repo in a tmp directory +# put the repo in a tmp directory tmpdir="$(mktemp -d)" trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -pushd "${tmpdir}" -git clone -b $BRANCH $GIT_REPO -trap "rm -rf $GIT_BASE_DIR" EXIT -cd "$GIT_BASE_DIR" -git checkout $VERSION -popd - -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" diff --git a/xds/third_party/protoc-gen-validate/import.sh b/xds/third_party/protoc-gen-validate/import.sh index 4e30b0e1180..64c92b16c20 100755 --- a/xds/third_party/protoc-gen-validate/import.sh +++ b/xds/third_party/protoc-gen-validate/import.sh @@ -13,33 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update GIT_ORIGIN_REV_ID then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=main -# import GIT_ORIGIN_REV_ID from one of the google internal CLs -GIT_ORIGIN_REV_ID=dfcdc5ea103dda467963fb7079e4df28debcfd28 -GIT_REPO="https://github.com/envoyproxy/protoc-gen-validate.git" -GIT_BASE_DIR=protoc-gen-validate -SOURCE_PROTO_BASE_DIR=protoc-gen-validate +# import VERSION from one of the google internal CLs +VERSION=dfcdc5ea103dda467963fb7079e4df28debcfd28 +DOWNLOAD_URL="https://github.com/envoyproxy/protoc-gen-validate/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="protoc-gen-validate-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( validate/validate.proto ) -# clone the protoc-gen-validate github repo in a tmp directory +pushd `git rev-parse --show-toplevel`/xds/third_party/protoc-gen-validate + +# put the repo in a tmp directory tmpdir="$(mktemp -d)" -pushd "${tmpdir}" -rm -rf "$GIT_BASE_DIR" -git clone -b $BRANCH $GIT_REPO -cd "$GIT_BASE_DIR" -git checkout $GIT_ORIGIN_REV_ID -popd +trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE -cp -p "${tmpdir}/${GIT_BASE_DIR}/NOTICE" NOTICE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/NOTICE" NOTICE +rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" pushd "${TARGET_PROTO_BASE_DIR}" @@ -51,4 +49,4 @@ do done popd -rm -rf "$tmpdir" +popd diff --git a/xds/third_party/xds/import.sh b/xds/third_party/xds/import.sh index f759cb0d35f..cda86d0368f 100755 --- a/xds/third_party/xds/import.sh +++ b/xds/third_party/xds/import.sh @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update VERSION then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=main # import VERSION from one of the google internal CLs VERSION=e9ce68804cb4e64cab5a52e3c8baf840d4ff87b7 -GIT_REPO="https://github.com/cncf/xds.git" -GIT_BASE_DIR=xds -SOURCE_PROTO_BASE_DIR=xds +DOWNLOAD_URL="https://github.com/cncf/xds/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="xds-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( @@ -54,18 +53,12 @@ xds/type/v3/typed_struct.proto pushd `git rev-parse --show-toplevel`/xds/third_party/xds -# clone the xds github repo in a tmp directory +# put the repo in a tmp directory tmpdir="$(mktemp -d)" -trap "rm -rf $tmpdir" EXIT +trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -pushd "${tmpdir}" -git clone -b $BRANCH $GIT_REPO -trap "rm -rf $GIT_BASE_DIR" EXIT -cd "$GIT_BASE_DIR" -git checkout $VERSION -popd - -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" From f90656293f00bca47193f474c2a2284cda21aab7 Mon Sep 17 00:00:00 2001 From: Larry Safran Date: Tue, 15 Aug 2023 17:33:23 -0700 Subject: [PATCH 14/14] Mark MultiChildLoadBalancer as Internal. (#10481) * Mark MultiChildLoadBalancer as Internal. Cannot move to the internal package because of its use of classes in the util package. * Exclude MultiChildLoadBalancer from javadoc generation. * Fix javadoc creation. --- all/build.gradle | 1 + gae-interop-testing/gae-jdk8/build.gradle | 4 ++++ util/build.gradle | 4 ++++ util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java | 6 +++--- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/all/build.gradle b/all/build.gradle index fe9ba187980..dceee473316 100644 --- a/all/build.gradle +++ b/all/build.gradle @@ -22,6 +22,7 @@ def subprojects = [ project(':grpc-servlet-jakarta'), project(':grpc-stub'), project(':grpc-testing'), + project(':grpc-util'), project(':grpc-xds'), ] diff --git a/gae-interop-testing/gae-jdk8/build.gradle b/gae-interop-testing/gae-jdk8/build.gradle index f3ff765ddfb..81aeda54a49 100644 --- a/gae-interop-testing/gae-jdk8/build.gradle +++ b/gae-interop-testing/gae-jdk8/build.gradle @@ -166,3 +166,7 @@ tasks.register("runInteropTestRemote") { throw new GradleException("Interop test failed:\nthrowable:${caught}") } } + +tasks.named("javadoc").configure { + enabled = false +} diff --git a/util/build.gradle b/util/build.gradle index af4e6044ef2..6234bdc8f86 100644 --- a/util/build.gradle +++ b/util/build.gradle @@ -37,3 +37,7 @@ animalsniffer { sourceSets.test ] } + +tasks.named("javadoc").configure { + exclude 'io/grpc/util/MultiChildLoadBalancer.java' +} diff --git a/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java b/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java index 3671505a345..be0a23a1642 100644 --- a/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java +++ b/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java @@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import io.grpc.ConnectivityState; +import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancerProvider; import io.grpc.Status; @@ -40,10 +41,9 @@ /** * A base load balancing policy for those policies which has multiple children such as - * ClusterManager or the petiole policies. - * - * @since 1.58 + * ClusterManager or the petiole policies. For internal use only. */ +@Internal public abstract class MultiChildLoadBalancer extends LoadBalancer { @VisibleForTesting