diff --git a/RELEASING.md b/RELEASING.md index 17ea03bff94..203daec4aa6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -174,7 +174,7 @@ Tagging the Release tools/interop_matrix/testcases/java__master # Commit the changes - git commit --all -m "Add grpc-java $MAJOR.$MINOR.$PATCH to client_matrix.py" + git commit --all -m "[interop] Add grpc-java $MAJOR.$MINOR.$PATCH to client_matrix.py" # Create a PR with the `release notes: no` label and run ad-hoc test against your PR ``` diff --git a/all/build.gradle b/all/build.gradle index fe9ba187980..dceee473316 100644 --- a/all/build.gradle +++ b/all/build.gradle @@ -22,6 +22,7 @@ def subprojects = [ project(':grpc-servlet-jakarta'), project(':grpc-stub'), project(':grpc-testing'), + project(':grpc-util'), project(':grpc-xds'), ] diff --git a/android-interop-testing/build.gradle b/android-interop-testing/build.gradle index feccf6040bb..69f5a62bfaa 100644 --- a/android-interop-testing/build.gradle +++ b/android-interop-testing/build.gradle @@ -88,8 +88,7 @@ dependencies { compileOnly libraries.javax.annotation - androidTestImplementation project(':grpc-netty'), - 'androidx.test.ext:junit:1.1.3', + androidTestImplementation 'androidx.test.ext:junit:1.1.3', 'androidx.test:runner:1.4.0' } diff --git a/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java b/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java index f002dd291c7..f5e54da5d4e 100644 --- a/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java +++ b/android-interop-testing/src/androidTest/java/io/grpc/android/integrationtest/UdsChannelInteropTest.java @@ -22,9 +22,10 @@ import androidx.test.InstrumentationRegistry; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.rule.ActivityTestRule; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; import io.grpc.Server; import io.grpc.android.UdsChannelBuilder; -import io.grpc.netty.NettyServerBuilder; import io.grpc.testing.integration.TestServiceImpl; import java.io.IOException; import java.util.concurrent.ExecutionException; @@ -68,7 +69,7 @@ public void setUp() throws IOException { // Start local server. server = - NettyServerBuilder.forPort(0) + Grpc.newServerBuilderForPort(0, InsecureServerCredentials.create()) .maxInboundMessageSize(16 * 1024 * 1024) .addService(new TestServiceImpl(serverExecutor)) .build(); diff --git a/api/src/main/java/io/grpc/LoadBalancer.java b/api/src/main/java/io/grpc/LoadBalancer.java index 5617d279862..d7e3fbb917c 100644 --- a/api/src/main/java/io/grpc/LoadBalancer.java +++ b/api/src/main/java/io/grpc/LoadBalancer.java @@ -115,6 +115,19 @@ public abstract class LoadBalancer { @NameResolver.ResolutionResultAttr public static final Attributes.Key> ATTR_HEALTH_CHECKING_CONFIG = Attributes.Key.create("internal:health-checking-config"); + + public static final SubchannelPicker EMPTY_PICKER = new SubchannelPicker() { + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withNoResult(); + } + + @Override + public String toString() { + return "EMPTY_PICKER"; + } + }; + private int recursionCount; /** @@ -1398,4 +1411,26 @@ public abstract static class Factory { */ public abstract LoadBalancer newLoadBalancer(Helper helper); } + + public static final class ErrorPicker extends SubchannelPicker { + + private final Status error; + + public ErrorPicker(Status error) { + this.error = checkNotNull(error, "error"); + } + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withError(error); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("error", error) + .toString(); + } + } + } diff --git a/api/src/main/java/io/grpc/ServerRegistry.java b/api/src/main/java/io/grpc/ServerRegistry.java index e6a067ce87f..70fd3657307 100644 --- a/api/src/main/java/io/grpc/ServerRegistry.java +++ b/api/src/main/java/io/grpc/ServerRegistry.java @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -92,7 +93,7 @@ public static synchronized ServerRegistry getDefaultRegistry() { if (instance == null) { List providerList = ServiceProviders.loadAll( ServerProvider.class, - Collections.>emptyList(), + getHardCodedClasses(), ServerProvider.class.getClassLoader(), new ServerPriorityAccessor()); instance = new ServerRegistry(); @@ -119,6 +120,20 @@ ServerProvider provider() { return providers.isEmpty() ? null : providers.get(0); } + @VisibleForTesting + static List> getHardCodedClasses() { + // Class.forName(String) is used to remove the need for ProGuard configuration. Note that + // ProGuard does not detect usages of Class.forName(String, boolean, ClassLoader): + // https://sourceforge.net/p/proguard/bugs/418/ + List> list = new ArrayList<>(); + try { + list.add(Class.forName("io.grpc.okhttp.OkHttpServerProvider")); + } catch (ClassNotFoundException e) { + logger.log(Level.FINE, "Unable to find OkHttpServerProvider", e); + } + return Collections.unmodifiableList(list); + } + ServerBuilder newServerBuilderForPort(int port, ServerCredentials creds) { List providers = providers(); if (providers.isEmpty()) { diff --git a/api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java b/api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java new file mode 100644 index 00000000000..b15d4dfde19 --- /dev/null +++ b/api/src/testFixtures/java/io/grpc/ServerRegistryAccessor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc; + +/** Accesses test-only methods of {@link ServerRegistry}. */ +public final class ServerRegistryAccessor { + private ServerRegistryAccessor() {} + + public static Iterable> getHardCodedClasses() { + return ServerRegistry.getHardCodedClasses(); + } +} diff --git a/build.gradle b/build.gradle index e0a905f2452..4d9f3a62df0 100644 --- a/build.gradle +++ b/build.gradle @@ -417,31 +417,25 @@ subprojects { def baselineGrpcVersion = '1.6.1' // Get the baseline version's jar for this subproject - File baselineArtifact = null - // Use a detached configuration, otherwise the current version's jar will take precedence - // over the baseline jar. + configurations { + baselineArtifact + } // A necessary hack, the intuitive thing does NOT work: // https://discuss.gradle.org/t/is-the-default-configuration-leaking-into-independent-configurations/2088/6 def oldGroup = project.group try { project.group = 'virtual_group_for_japicmp' - String depModule = "io.grpc:${project.name}:${baselineGrpcVersion}@jar" - String depJar = "${project.name}-${baselineGrpcVersion}.jar" - Configuration configuration = configurations.detachedConfiguration( - dependencies.create(depModule) - ) - baselineArtifact = files(configuration.files).filter { - it.name.equals(depJar) - }.singleFile + dependencies { + baselineArtifact "io.grpc:${project.name}:${baselineGrpcVersion}@jar" + } } finally { project.group = oldGroup } // Add a japicmp task that compares the current .jar with baseline .jar tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) { - dependsOn jar - oldClasspath.from baselineArtifact - newClasspath.from jar.archiveFile + oldClasspath.from configurations.baselineArtifact + newClasspath.from tasks.named("jar") onlyBinaryIncompatibleModified = false // Be quiet about things that did not change onlyModified = true @@ -454,12 +448,10 @@ subprojects { // Also break on source incompatible changes, not just binary. // Eg adding abstract method to public class. - // TODO(zpencer): enable after japicmp-gradle-plugin/pull/14 - // breakOnSourceIncompatibility = true + failOnSourceIncompatibility = true // Ignore any classes or methods marked @ExperimentalApi - // TODO(zpencer): enable after japicmp-gradle-plugin/pull/15 - // annotationExcludes = ['@io.grpc.ExperimentalApi'] + annotationExcludes = ['@io.grpc.ExperimentalApi'] } } } diff --git a/core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java b/core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java new file mode 100644 index 00000000000..1ea61252e6b --- /dev/null +++ b/core/src/main/java/io/grpc/internal/PickFirstLeafLoadBalancer.java @@ -0,0 +1,477 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.IDLE; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.SHUTDOWN; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Lists; +import io.grpc.Attributes; +import io.grpc.ConnectivityState; +import io.grpc.ConnectivityStateInfo; +import io.grpc.EquivalentAddressGroup; +import io.grpc.ExperimentalApi; +import io.grpc.LoadBalancer; +import io.grpc.Status; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + +/** + * A {@link LoadBalancer} that provides no load-balancing over the addresses from the {@link + * io.grpc.NameResolver}. The channel's default behavior is used, which is walking down the address + * list and sticking to the first that works. + */ +@ExperimentalApi("https://github.com/grpc/grpc-java/issues/10383") +final class PickFirstLeafLoadBalancer extends LoadBalancer { + private final Helper helper; + private final Map subchannels = new HashMap<>(); + private Index addressIndex; + private ConnectivityState currentState = IDLE; + + PickFirstLeafLoadBalancer(Helper helper) { + this.helper = checkNotNull(helper, "helper"); + } + + @Override + public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { + List servers = resolvedAddresses.getAddresses(); + if (servers.isEmpty()) { + handleNameResolutionError(Status.UNAVAILABLE.withDescription( + "NameResolver returned no usable address. addrs=" + resolvedAddresses.getAddresses() + + ", attrs=" + resolvedAddresses.getAttributes())); + return false; + } + for (EquivalentAddressGroup eag : servers) { + if (eag == null) { + handleNameResolutionError(Status.UNAVAILABLE.withDescription( + "NameResolver returned address list with null endpoint. addrs=" + + resolvedAddresses.getAddresses() + ", attrs=" + resolvedAddresses.getAttributes())); + return false; + } + } + // We can optionally be configured to shuffle the address list. This can help better distribute + // the load. + if (resolvedAddresses.getLoadBalancingPolicyConfig() + instanceof PickFirstLeafLoadBalancerConfig) { + PickFirstLeafLoadBalancerConfig config + = (PickFirstLeafLoadBalancerConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); + if (config.shuffleAddressList != null && config.shuffleAddressList) { + servers = new ArrayList(servers); + Collections.shuffle(servers, + config.randomSeed != null ? new Random(config.randomSeed) : new Random()); + } + } + + final List newImmutableAddressGroups = + Collections.unmodifiableList(new ArrayList<>(servers)); + + if (addressIndex == null) { + addressIndex = new Index(newImmutableAddressGroups); + } else if (currentState == READY) { + // If a ready subchannel exists in new address list, + // keep this connection and don't create new subchannels + SocketAddress previousAddress = addressIndex.getCurrentAddress(); + addressIndex.updateGroups(newImmutableAddressGroups); + if (addressIndex.seekTo(previousAddress)) { + return true; + } + addressIndex.reset(); + } else { + addressIndex.updateGroups(newImmutableAddressGroups); + } + + // Create subchannels for all new addresses, preserving existing connections + Set oldAddrs = new HashSet<>(subchannels.keySet()); + Set newAddrs = new HashSet<>(); + for (EquivalentAddressGroup endpoint : newImmutableAddressGroups) { + for (SocketAddress addr : endpoint.getAddresses()) { + newAddrs.add(addr); + if (!subchannels.containsKey(addr)) { + createNewSubchannel(addr); + } + } + } + + // remove old subchannels that were not in new address list + for (SocketAddress oldAddr : oldAddrs) { + if (!newAddrs.contains(oldAddr)) { + subchannels.get(oldAddr).getSubchannel().shutdown(); + subchannels.remove(oldAddr); + } + } + + if (oldAddrs.size() == 0 || currentState == CONNECTING || currentState == READY) { + // start connection attempt at first address + updateBalancingState(CONNECTING, new Picker(PickResult.withNoResult())); + requestConnection(); + + } else if (currentState == IDLE) { + // start connection attempt at first address when requested + SubchannelPicker picker = new RequestConnectionPicker(this); + updateBalancingState(IDLE, picker); + + } else if (currentState == TRANSIENT_FAILURE) { + // start connection attempt at first address + requestConnection(); + } + + return true; + } + + @Override + public void handleNameResolutionError(Status error) { + for (SubchannelData subchannelData : subchannels.values()) { + subchannelData.getSubchannel().shutdown(); + } + subchannels.clear(); + // NB(lukaszx0) Whether we should propagate the error unconditionally is arguable. It's fine + // for time being. + updateBalancingState(TRANSIENT_FAILURE, new Picker(PickResult.withError(error))); + } + + void processSubchannelState(Subchannel subchannel, ConnectivityStateInfo stateInfo) { + ConnectivityState newState = stateInfo.getState(); + // Shutdown channels/previously relevant subchannels can still callback with state updates. + // To prevent pickers from returning these obselete subchannels, this logic + // is included to check if the current list of active subchannels includes this subchannel. + if (!subchannels.containsKey(getAddress(subchannel)) + || subchannels.get(getAddress(subchannel)).getSubchannel() != subchannel) { + return; + } + if (newState == SHUTDOWN) { + return; + } + if (newState == IDLE) { + helper.refreshNameResolution(); + } + // If we are transitioning from a TRANSIENT_FAILURE to CONNECTING or IDLE we ignore this state + // transition and still keep the LB in TRANSIENT_FAILURE state. This is referred to as "sticky + // transient failure". Only a subchannel state change to READY will get the LB out of + // TRANSIENT_FAILURE. If the state is IDLE we additionally request a new connection so that we + // keep retrying for a connection. + + // With the new pick first implementation, individual subchannels will have their own backoff + // on a per-address basis. Thus, iterative requests for connections will not be requested + // once the first pass through is complete. + // However, every time there is an address update, we will perform a pass through for the new + // addresses in the updated list. + subchannels.get(getAddress(subchannel)).updateState(newState); + if (currentState == TRANSIENT_FAILURE) { + if (newState == CONNECTING) { + // each subchannel is responsible for its own backoff + return; + } else if (newState == IDLE) { + requestConnection(); + return; + } + } + + switch (newState) { + case IDLE: + // Shutdown when ready: connect from beginning when prompted + addressIndex.reset(); + updateBalancingState(IDLE, new RequestConnectionPicker(this));; + break; + case CONNECTING: + // It's safe to use RequestConnectionPicker here, so when coming from IDLE we could leave + // the current picker in-place. But ignoring the potential optimization is simpler. + updateBalancingState(CONNECTING, new Picker(PickResult.withNoResult())); + break; + case READY: + updateBalancingState(READY, new Picker(PickResult.withSubchannel(subchannel))); + shutdownRemaining(subchannel); + addressIndex.seekTo(getAddress(subchannel)); + break; + case TRANSIENT_FAILURE: + // If we are looking at current channel, request a connection if possible + if (addressIndex.isValid() + && subchannels.get(addressIndex.getCurrentAddress()).getSubchannel() == subchannel) { + addressIndex.increment(); + requestConnection(); + + // If no addresses remaining, go into TRANSIENT_FAILURE + if (!addressIndex.isValid()) { + helper.refreshNameResolution(); + updateBalancingState(TRANSIENT_FAILURE, + new Picker(PickResult.withError(stateInfo.getStatus()))); + } + } + break; + default: + throw new IllegalArgumentException("Unsupported state:" + newState); + } + } + + private void updateBalancingState(ConnectivityState state, SubchannelPicker picker) { + if (state != currentState || state == READY || state == TRANSIENT_FAILURE) { + currentState = state; + helper.updateBalancingState(state, picker); + } + } + + @Override + public void shutdown() { + for (SubchannelData subchannelData : subchannels.values()) { + subchannelData.getSubchannel().shutdown(); + } + subchannels.clear(); + } + + /** + * Shuts down remaining subchannels. Called when a subchannel becomes ready, which means + * that all other subchannels must be shutdown. + */ + private void shutdownRemaining(Subchannel activeSubchannel) { + for (SubchannelData subchannelData : subchannels.values()) { + if (!subchannelData.getSubchannel().equals(activeSubchannel)) { + subchannelData.getSubchannel().shutdown(); + } + } + subchannels.clear(); + subchannels.put(getAddress(activeSubchannel), new SubchannelData(activeSubchannel, READY)); + } + + /** + * Requests a connection to the next applicable address' subchannel, creating one if necessary + * If the current channel has already attempted a connection, we attempt a connection + * to the next address/subchannel in our list. + */ + @Override + public void requestConnection() { + if (subchannels.size() == 0) { + return; + } + if (addressIndex.isValid()) { + Subchannel subchannel = subchannels.containsKey(addressIndex.getCurrentAddress()) + ? subchannels.get(addressIndex.getCurrentAddress()).getSubchannel() + : createNewSubchannel(addressIndex.getCurrentAddress()); + + ConnectivityState subchannelState = + subchannels.get(addressIndex.getCurrentAddress()).getState(); + if (subchannelState == IDLE) { + subchannel.requestConnection(); + } else if (subchannelState == CONNECTING || subchannelState == TRANSIENT_FAILURE) { + addressIndex.increment(); + requestConnection(); + } + } + } + + private Subchannel createNewSubchannel(SocketAddress addr) { + final Subchannel subchannel = helper.createSubchannel( + CreateSubchannelArgs.newBuilder() + .setAddresses(Lists.newArrayList( + new EquivalentAddressGroup(addr))) + .build()); + subchannels.put(addr, new SubchannelData(subchannel, IDLE)); + subchannel.start(new SubchannelStateListener() { + @Override + public void onSubchannelState(ConnectivityStateInfo stateInfo) { + processSubchannelState(subchannel, stateInfo); + } + }); + return subchannel; + } + + private SocketAddress getAddress(Subchannel subchannel) { + return subchannel.getAddresses().getAddresses().get(0); + } + + @VisibleForTesting + ConnectivityState getCurrentState() { + return this.currentState; + } + + /** + * No-op picker which doesn't add any custom picking logic. It just passes already known result + * received in constructor. + */ + private static final class Picker extends SubchannelPicker { + private final PickResult result; + + Picker(PickResult result) { + this.result = checkNotNull(result, "result"); + } + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return result; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(Picker.class).add("result", result).toString(); + } + } + + /** + * Picker that requests connection during the first pick, and returns noResult. + */ + private final class RequestConnectionPicker extends SubchannelPicker { + private final PickFirstLeafLoadBalancer pickFirstLeafLoadBalancer; + private final AtomicBoolean connectionRequested = new AtomicBoolean(false); + + RequestConnectionPicker(PickFirstLeafLoadBalancer pickFirstLeafLoadBalancer) { + this.pickFirstLeafLoadBalancer = + checkNotNull(pickFirstLeafLoadBalancer, "pickFirstLeafLoadBalancer"); + } + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + if (connectionRequested.compareAndSet(false, true)) { + helper.getSynchronizationContext().execute(new Runnable() { + @Override + public void run() { + pickFirstLeafLoadBalancer.requestConnection(); + } + }); + } + return PickResult.withNoResult(); + } + } + + /** + * Index as in 'i', the pointer to an entry. Not a "search index." + */ + @VisibleForTesting + static final class Index { + private List addressGroups; + private int groupIndex; + private int addressIndex; + + public Index(List groups) { + this.addressGroups = groups; + } + + public boolean isValid() { + // addressIndex will never be invalid + return groupIndex < addressGroups.size(); + } + + public boolean isAtBeginning() { + return groupIndex == 0 && addressIndex == 0; + } + + public void increment() { + EquivalentAddressGroup group = addressGroups.get(groupIndex); + addressIndex++; + if (addressIndex >= group.getAddresses().size()) { + groupIndex++; + addressIndex = 0; + } + } + + public void reset() { + groupIndex = 0; + addressIndex = 0; + } + + public SocketAddress getCurrentAddress() { + return addressGroups.get(groupIndex).getAddresses().get(addressIndex); + } + + public Attributes getCurrentEagAttributes() { + return addressGroups.get(groupIndex).getAttributes(); + } + + public List getGroups() { + return addressGroups; + } + + /** + * Update to new groups, resetting the current index. + */ + public void updateGroups(List newGroups) { + addressGroups = newGroups; + reset(); + } + + /** + * Returns false if the needle was not found and the current index was left unchanged. + */ + public boolean seekTo(SocketAddress needle) { + for (int i = 0; i < addressGroups.size(); i++) { + EquivalentAddressGroup group = addressGroups.get(i); + int j = group.getAddresses().indexOf(needle); + if (j == -1) { + continue; + } + this.groupIndex = i; + this.addressIndex = j; + return true; + } + return false; + } + } + + private static final class SubchannelData { + private final Subchannel subchannel; + private ConnectivityState state; + + public SubchannelData(Subchannel subchannel, ConnectivityState state) { + this.subchannel = subchannel; + this.state = state; + } + + public Subchannel getSubchannel() { + return this.subchannel; + } + + public ConnectivityState getState() { + return this.state; + } + + private void updateState(ConnectivityState newState) { + this.state = newState; + } + } + + public static final class PickFirstLeafLoadBalancerConfig { + + @Nullable + public final Boolean shuffleAddressList; + + // For testing purposes only, not meant to be parsed from a real config. + @Nullable + final Long randomSeed; + + public PickFirstLeafLoadBalancerConfig(@Nullable Boolean shuffleAddressList) { + this(shuffleAddressList, null); + } + + PickFirstLeafLoadBalancerConfig(@Nullable Boolean shuffleAddressList, + @Nullable Long randomSeed) { + this.shuffleAddressList = shuffleAddressList; + this.randomSeed = randomSeed; + } + } +} diff --git a/core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java b/core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java new file mode 100644 index 00000000000..5437cbc9198 --- /dev/null +++ b/core/src/test/java/io/grpc/internal/PickFirstLeafLoadBalancerTest.java @@ -0,0 +1,2131 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.IDLE; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; +import io.grpc.Attributes; +import io.grpc.ConnectivityState; +import io.grpc.ConnectivityStateInfo; +import io.grpc.EquivalentAddressGroup; +import io.grpc.LoadBalancer.CreateSubchannelArgs; +import io.grpc.LoadBalancer.Helper; +import io.grpc.LoadBalancer.PickResult; +import io.grpc.LoadBalancer.PickSubchannelArgs; +import io.grpc.LoadBalancer.ResolvedAddresses; +import io.grpc.LoadBalancer.Subchannel; +import io.grpc.LoadBalancer.SubchannelPicker; +import io.grpc.LoadBalancer.SubchannelStateListener; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; +import io.grpc.internal.PickFirstLeafLoadBalancer.PickFirstLeafLoadBalancerConfig; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + + +/** Unit test for {@link PickFirstLeafLoadBalancer}. */ +@RunWith(JUnit4.class) +public class PickFirstLeafLoadBalancerTest { + private PickFirstLeafLoadBalancer loadBalancer; + private final List servers = Lists.newArrayList(); + private static final Attributes.Key FOO = Attributes.Key.create("foo"); + + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + private final Attributes affinity = Attributes.newBuilder().set(FOO, "bar").build(); + @Rule + public final MockitoRule mocks = MockitoJUnit.rule(); + @Captor + private ArgumentCaptor pickerCaptor; + @Captor + private ArgumentCaptor connectivityStateCaptor; + @Captor + private ArgumentCaptor createArgsCaptor; + @Captor + private ArgumentCaptor stateListenerCaptor; + @Mock + private Helper mockHelper; + @Mock + private FakeSubchannel mockSubchannel1; + @Mock + private FakeSubchannel mockSubchannel2; + @Mock + private FakeSubchannel mockSubchannel3; + @Mock + private FakeSubchannel mockSubchannel4; + @Mock // This LoadBalancer doesn't use any of the arg fields, as verified in tearDown(). + private PickSubchannelArgs mockArgs; + + @Before + public void setUp() { + for (int i = 1; i < 5; i++) { + SocketAddress addr = new FakeSocketAddress("server" + i); + servers.add(new EquivalentAddressGroup(addr)); + } + mockSubchannel1 = new FakeSubchannel(Lists.newArrayList( + new EquivalentAddressGroup(new FakeSocketAddress("fake"))), null); + mockSubchannel1 = mock(FakeSubchannel.class); + mockSubchannel2 = mock(FakeSubchannel.class); + mockSubchannel3 = mock(FakeSubchannel.class); + mockSubchannel4 = mock(FakeSubchannel.class); + when(mockHelper.createSubchannel(any(CreateSubchannelArgs.class))) + .thenReturn(mockSubchannel1, mockSubchannel2, mockSubchannel3, mockSubchannel4); + + when(mockSubchannel1.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + when(mockSubchannel2.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(1))); + when(mockSubchannel3.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(2))); + when(mockSubchannel4.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(3))); + + when(mockHelper.getSynchronizationContext()).thenReturn(syncContext); + loadBalancer = new PickFirstLeafLoadBalancer(mockHelper); + } + + @After + public void tearDown() throws Exception { + verifyNoMoreInteractions(mockArgs); + } + + @Test + public void pickAfterResolved() throws Exception { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolved_shuffle() throws Exception { + servers.remove(3); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity) + .setLoadBalancingPolicyConfig(new PickFirstLeafLoadBalancerConfig(true, 123L)).build()); + + verify(mockHelper, times(3)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + // We should still see the same set of addresses. + // Because we use a fixed seed, the addresses should always be shuffled in this order. + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolved_noShuffle() throws Exception { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity) + .setLoadBalancingPolicyConfig(new PickFirstLeafLoadBalancerConfig(false)).build()); + + verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void requestConnectionPicker() throws Exception { + // Set up + assertEquals(IDLE, loadBalancer.getCurrentState()); + List newServers = Lists.newArrayList(servers.get(0), servers.get(1), + servers.get(2)); + + // Accepting resolved addresses + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, mockSubchannel3); + + // We initialize and start all subchannels + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + + // We start connection attempt to the first address in the list + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // If we send the first subchannel into idle ... + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + + SubchannelPicker picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() requests a connection, gives the same result when called twice. + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once for a total of two connection requests. + inOrder.verify(mockSubchannel1).requestConnection(); + verify(mockSubchannel1, times(2)).requestConnection(); + } + + @Test + public void refreshNameResolutionAfterSubchannelConnectionBroken() { + List newServers = Lists.newArrayList(servers.get(0)); + when(mockSubchannel1.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + + // accept resolved addresses + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + InOrder inOrder = inOrder(mockHelper, mockSubchannel1); + verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertNull(pickerCaptor.getValue().pickSubchannel(mockArgs).getSubchannel()); + inOrder.verify(mockSubchannel1).requestConnection(); + + Status error = Status.UNAUTHENTICATED.withDescription("permission denied"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + assertEquals(mockSubchannel1, pickerCaptor.getValue().pickSubchannel(mockArgs).getSubchannel()); + + // Simulate receiving go-away so the subchannel transit to IDLE. + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), any(SubchannelPicker.class)); + } + + @Test + public void pickAfterResolvedAndUnchanged() throws Exception { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + verify(mockSubchannel1).requestConnection(); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockSubchannel1).requestConnection(); + + verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + assertThat(createArgsCaptor.getValue()).isNotNull(); + verify(mockHelper) + .updateBalancingState(isA(ConnectivityState.class), isA(SubchannelPicker.class)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolvedAndChanged() throws Exception { + SocketAddress socketAddr1 = new FakeSocketAddress("oldserver"); + List oldServers = + Lists.newArrayList(new EquivalentAddressGroup(socketAddr1)); + + SocketAddress socketAddr2 = new FakeSocketAddress("newserver"); + List newServers = + Lists.newArrayList(new EquivalentAddressGroup(socketAddr2)); + + InOrder inOrder = inOrder(mockHelper, mockSubchannel1); + + // accept resolved addresses + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1).start(any(SubchannelStateListener.class)); + + // start connection attempt to first address + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + assertNull(pickerCaptor.getValue().pickSubchannel(mockArgs).getSubchannel()); + + // updating the subchannel addresses is unnecessary, but doesn't hurt anything + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + verify(mockSubchannel1).shutdown(); + + verifyNoMoreInteractions(mockSubchannel1); + verify(mockSubchannel2).start(any(SubchannelStateListener.class)); + } + + @Test + public void pickAfterStateChangeAfterResolution() throws Exception { + InOrder inOrder = inOrder(mockHelper); + + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + inOrder.verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + reset(mockHelper); + when(mockHelper.getSynchronizationContext()).thenReturn(syncContext); + + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + + // subchannel reports connecting when pick subchannel is called + assertEquals(Status.OK, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + + Status error = Status.UNAVAILABLE.withDescription("boom!"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + stateListener4.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(mockSubchannel1, picker.pickSubchannel(mockArgs).getSubchannel()); + + verify(mockHelper, atLeast(0)).getSynchronizationContext(); // Don't care + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void pickAfterResolutionAfterTransientValue() throws Exception { + InOrder inOrder = inOrder(mockHelper); + List newServers = Lists.newArrayList(servers.get(0)); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + reset(mockHelper); + when(mockHelper.getSynchronizationContext()).thenReturn(syncContext); + + // An error has happened. + Status error = Status.UNAVAILABLE.withDescription("boom!"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + inOrder.verify(mockHelper).refreshNameResolution(); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + + // Transition from TRANSIENT_ERROR to CONNECTING should also be ignored. + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + verifyNoMoreInteractions(mockHelper); + assertEquals(error, pickerCaptor.getValue().pickSubchannel(mockArgs).getStatus()); + } + + @Test + public void nameResolutionError() throws Exception { + Status error = Status.NOT_FOUND.withDescription("nameResolutionError"); + loadBalancer.handleNameResolutionError(error); + verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + PickResult pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertNull(pickResult.getSubchannel()); + assertEquals(error, pickResult.getStatus()); + verify(mockSubchannel1, never()).requestConnection(); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void nameResolutionError_emptyAddressList() throws Exception { + servers.clear(); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + verify(mockHelper).updateBalancingState(connectivityStateCaptor.capture(), + pickerCaptor.capture()); + PickResult pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertThat(pickResult.getSubchannel()).isNull(); + assertThat(pickResult.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(pickResult.getStatus().getDescription()).contains("returned no usable address"); + verify(mockSubchannel1, never()).requestConnection(); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void nameResolutionSuccessAfterError() throws Exception { + InOrder inOrder = inOrder(mockHelper); + + loadBalancer.handleNameResolutionError(Status.NOT_FOUND.withDescription("nameResolutionError")); + inOrder.verify(mockHelper) + .updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class)); + verify(mockSubchannel1, never()).requestConnection(); + + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + inOrder.verify(mockHelper, times(4)).createSubchannel(createArgsCaptor.capture()); + List argsList = createArgsCaptor.getAllValues(); + assertThat(argsList.get(0).getAddresses().get(0)).isEqualTo(servers.get(0)); + assertThat(argsList.get(1).getAddresses().get(0)).isEqualTo(servers.get(1)); + assertThat(argsList.get(2).getAddresses().get(0)).isEqualTo(servers.get(2)); + assertThat(argsList.get(3).getAddresses().get(0)).isEqualTo(servers.get(3)); + assertThat(argsList.get(0).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(1).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(2).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(3).getAddresses().size()).isEqualTo(1); + assertThat(argsList.get(0).getAttributes()).isEqualTo(Attributes.EMPTY); + assertThat(argsList.get(1).getAttributes()).isEqualTo(Attributes.EMPTY); + assertThat(argsList.get(2).getAttributes()).isEqualTo(Attributes.EMPTY); + assertThat(argsList.get(3).getAttributes()).isEqualTo(Attributes.EMPTY); + + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + verify(mockSubchannel1).requestConnection(); + + assertNull(pickerCaptor.getValue().pickSubchannel(mockArgs) + .getSubchannel()); + + assertEquals(pickerCaptor.getValue().pickSubchannel(mockArgs), + pickerCaptor.getValue().pickSubchannel(mockArgs)); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void nameResolutionErrorWithStateChanges() throws Exception { + List newServers = Lists.newArrayList(servers.get(0)); + InOrder inOrder = inOrder(mockHelper); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState( + eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + Status error = Status.NOT_FOUND.withDescription("nameResolutionError"); + loadBalancer.handleNameResolutionError(error); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + PickResult pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertNull(pickResult.getSubchannel()); + assertEquals(error, pickResult.getStatus()); + + Status error2 = Status.NOT_FOUND.withDescription("nameResolutionError2"); + loadBalancer.handleNameResolutionError(error2); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + + pickResult = pickerCaptor.getValue().pickSubchannel(mockArgs); + assertNull(pickResult.getSubchannel()); + assertEquals(error2, pickResult.getStatus()); + + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void requestConnection() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // calling requestConnection() starts next subchannel + loadBalancer.requestConnection(); + inOrder.verify(mockSubchannel2).requestConnection(); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // calling requestConnection is now a no-op + loadBalancer.requestConnection(); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void updateAddresses_emptyEagList_returns_false() { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + assertFalse(loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Arrays.asList()).setAttributes(affinity).build())); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + } + + @Test + public void updateAddresses_eagListWithNull_returns_false() { + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + List eags = Arrays.asList((EquivalentAddressGroup) null); + assertFalse(loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(eags).setAttributes(affinity).build())); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + } + + @Test + public void updateAddresses_disjoint_idle() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Going into IDLE state + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + + // Creating second set of disjoint endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // We create new channels, remove old ones, and keep intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + verify(mockSubchannel1).shutdown(); + verify(mockSubchannel2).shutdown(); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel3).requestConnection(); + + // Ready subchannel 3 + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + + // Picking a subchannel returns subchannel 3 + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_disjoint_connecting() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Old subchannels should shut down (in no particular order) and request a connection + verify(mockSubchannel1).shutdown(); + verify(mockSubchannel2).shutdown(); + inOrder.verify(mockSubchannel3).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // If old subchannel becomes ready, the state should not be affected + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Fail connection attempt to third address + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Verify starting connection attempt to fourth address + inOrder.verify(mockSubchannel4).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Succeed connection attempt to fourth address + stateListener4.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_disjoint_ready_twice() { + when(mockHelper.createSubchannel(any(CreateSubchannelArgs.class))) + .thenReturn(mockSubchannel1, mockSubchannel2, mockSubchannel3, + mockSubchannel4, mockSubchannel1, mockSubchannel2); + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection shuts down other subchannel + inOrder.verify(mockSubchannel2).shutdown(); + + // Verify that picker returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).shutdown(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel3).requestConnection(); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withNoResult(), picker.pickSubchannel(mockArgs)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Ready subchannel 3 + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection shuts down other subchannel + inOrder.verify(mockSubchannel4).shutdown(); + + // Verify that pickSubchannel() returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Creating third set of endpoints/addresses + List newestServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Second address update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newestServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel3).shutdown(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(PickResult.withNoResult(), pickerCaptor.getValue().pickSubchannel(mockArgs)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener4.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Connection attempt to address 1 is unsuccessful + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting connection attempt to address 2 + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Connection attempt to address 2 is successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection shuts down other subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel1).shutdown(); + + // Verify that picker still returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_disjoint_transient_failure() { + // Starting first connection attempt + when(mockHelper.createSubchannel(any(CreateSubchannelArgs.class))) + .thenReturn(mockSubchannel1, mockSubchannel2, mockSubchannel3, + mockSubchannel4, mockSubchannel1, mockSubchannel2); + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List addrs = Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); // sticky transient failure + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(2), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // subchannel 3 still attempts a connection even though we stay in transient failure + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener4 = stateListenerCaptor.getValue(); + verify(mockSubchannel1).shutdown(); + verify(mockSubchannel2).shutdown(); + inOrder.verify(mockSubchannel3).requestConnection(); + + // Obselete subchannels should not affect us + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Third subchannel connection attempt is unsuccessful + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + inOrder.verify(mockSubchannel4).requestConnection(); + + // Fourth subchannel connection attempt is successful + stateListener4.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Picking a subchannel returns subchannel 3 + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel3).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_idle() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // First connection attempt is successful + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Successful connection attempt shuts down other subchannels + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + + // Verify that picker returns correct subchannel + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Going into IDLE state, nothing should happen unless requested + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + verifyNoMoreInteractions(mockHelper); + + // Creating second set of intersecting endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // We create new channels and remove old ones, keeping intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + // Calling pickSubchannel() twice gave the same result + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + + // But the picker calls requestConnection() only once + inOrder.verify(mockSubchannel1).requestConnection(); + + // internal subchannel calls back and reports connecting + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(PickResult.withNoResult(), pickerCaptor.getValue().pickSubchannel(mockArgs)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Ready subchannel 1 + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + + // Picking a subchannel returns subchannel 1 + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_connecting() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(2)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // callback from internal subchannel + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // Creating second set of endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(3)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Don't unnecessarily create new subchannels and keep intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel4).start(stateListenerCaptor.capture()); + verifyNoMoreInteractions(mockHelper); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_ready() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Creating second set of endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(2)); + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // The state is still READY after update since we had an intersecting subchannel that was READY. + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker still returns correct subchannel + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_transient_failure() { + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List oldServers = + Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); // sticky transient failure + + // Creating second set of endpoints/addresses + List newServers = + Lists.newArrayList(servers.get(0), servers.get(1), servers.get(2)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // subchannel 3 still attempts a connection even though we stay in transient failure + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + inOrder.verify(mockSubchannel3).requestConnection(); + + // no other connections should be requested by LB, we should come out of backoff to request + verifyNoMoreInteractions(mockSubchannel3); + + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Second subchannel connection attempt is now successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + + // Picking a subchannel returns subchannel 3 + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel3).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_intersecting_enter_transient_failure() { + // after an address update occurs, verify that the client properly tries all + // addresses and only then enters transient failure. + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + + // callback from internal subchannel + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + + // Creating second set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(0), servers.get(2)); + + // Accept new resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // We create new channels and remove old ones, keeping intersecting ones + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + + // If obselete subchannel becomes ready, the state should not be affected + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is unsuccessful + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Subchannel 3 attempt starts but fails + inOrder.verify(mockSubchannel3).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + } + + @Test + public void updateAddresses_identical_idle() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Going into IDLE state + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + assertEquals(IDLE, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(3)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + + // First connection attempt is successful + assertEquals(IDLE, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_identical_connecting() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(2)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + + // First connection attempt is successful + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void updateAddresses_identical_ready() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is successful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel2).shutdown(); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(2)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker hasn't changed via checking mock helper's interactions + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void updateAddresses_identical_transient_failure() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is unsuccessful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second connection attempt is unsuccessful + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Accept same resolved addresses to update + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + verify(mockHelper, times(2)).createSubchannel(createArgsCaptor.capture()); + verify(mockSubchannel1, times(1)).start(stateListenerCaptor.capture()); + verify(mockSubchannel2, times(1)).start(stateListenerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // No new connections are requested, subchannels responsible for completing their own backoffs + verifyNoMoreInteractions(mockHelper); + + // First connection attempt is successful + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + inOrder.verify(mockSubchannel2).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void twoAddressesSeriallyConnect() { + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, mockSubchannel3); + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(servers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second connection attempt is successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void multiple_backoffs() { + // This test case mimics a backoff without implementing one + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List newServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Starting first connection attempt + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for second address + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Failing first connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for second address + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Connection attempt to second address is now successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + when(mockSubchannel2.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + inOrder.verify(mockSubchannel1).shutdown(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // If first subchannel is ready before it completes shutdown, we still choose subchannel 2 + // This can be verified by checking the mock helper. + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void backoff_faster_than_serial_connection() { + // this tests the case where a subchannel completes its backoff and readies a connection + // before the other addresses have a chance to complete their connection attempts + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = Lists.newArrayList(servers.get(0), + servers.get(1)); + + // Accepting resolved addresses starts all subchannels + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Connection attempt to first address is now successful + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void success_from_transient_failure() { + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accepting resolved addresses starts all subchannels + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Mimic backoff for first address + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + inOrder.verify(mockHelper).refreshNameResolution(); + inOrder.verify(mockHelper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); // sticky transient failure + + // Failing connection attempt to first address + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Mimic backoff for second address + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Connection attempt to second address is now successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + + // If first address is successful, nothing happens. Verify by checking mock helper + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + verifyNoMoreInteractions(mockHelper); + } + + @Test + public void lastAddressFailingNotTransientFailure() { + // This tests the case where after an address update, the last address escapes a backoff + // and reports transient failure before all addresses have failed connection + // attempts, in which case we should not report TRANSIENT_FAILURE. + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); + // Creating first set of endpoints/addresses + List oldServers = Lists.newArrayList(servers.get(0), servers.get(1)); + + // Accept Addresses and verify proper connection flow + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(oldServers).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // First connection attempt is unsuccessful + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second connection attempt is connecting + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Accept same resolved addresses to update + List newServers = Lists.newArrayList(servers.get(2), servers.get(1)); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(newServers).setAttributes(affinity).build()); + + // Verify that no new subchannels were created or started + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel1).shutdown(); + inOrder.verify(mockSubchannel3).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Second address connection attempt is unsuccessful, but should not go into transient failure + stateListener2.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Third address connection attempt is unsuccessful, now we enter transient failure + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Obselete subchannels have no impact + stateListener.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(TRANSIENT_FAILURE, loadBalancer.getCurrentState()); + + // Second subchannel is successful + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel3).shutdown(); + SubchannelPicker picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void recreate_shutdown_subchannel() { + // Take the case where the latter subchannel is readied. If we then go to an IDLE state and + // re-request a connection, we should start and create a new subchannel for the first + // address in our list. + + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = + Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Successful second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockSubchannel1).shutdown(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Go to IDLE + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + SubchannelPicker picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() requests a connection, gives the same result when called twice. + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel3).requestConnection(); + when(mockSubchannel3.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // first subchannel connection attempt fails + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // second subchannel connection attempt + inOrder.verify(mockSubchannel2).requestConnection(); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel3).shutdown(); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + @Test + public void ready_then_transient_failure_again() { + // Starting first connection attempt + InOrder inOrder = inOrder(mockHelper, mockSubchannel1, mockSubchannel2, + mockSubchannel3, mockSubchannel4); // captor: captures + + // Creating first set of endpoints/addresses + List addrs = + Lists.newArrayList(servers.get(0), servers.get(1)); + + assertEquals(IDLE, loadBalancer.getCurrentState()); + loadBalancer.acceptResolvedAddresses( + ResolvedAddresses.newBuilder().setAddresses(addrs).setAttributes(affinity).build()); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener = stateListenerCaptor.getValue(); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel2).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener2 = stateListenerCaptor.getValue(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel1).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Failing first connection attempt + Status error = Status.UNAVAILABLE.withDescription("Simulated connection error"); + stateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Starting second connection attempt + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + inOrder.verify(mockSubchannel2).requestConnection(); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // Successful second connection attempt + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + inOrder.verify(mockSubchannel1).shutdown(); + assertEquals(READY, loadBalancer.getCurrentState()); + + // Go to IDLE + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(IDLE)); + inOrder.verify(mockHelper).updateBalancingState(eq(IDLE), pickerCaptor.capture()); + assertEquals(IDLE, loadBalancer.getCurrentState()); + + SubchannelPicker picker = pickerCaptor.getValue(); + + // Calling pickSubchannel() requests a connection, gives the same result when called twice. + assertEquals(picker.pickSubchannel(mockArgs), picker.pickSubchannel(mockArgs)); + inOrder.verify(mockHelper).createSubchannel(createArgsCaptor.capture()); + inOrder.verify(mockSubchannel3).start(stateListenerCaptor.capture()); + SubchannelStateListener stateListener3 = stateListenerCaptor.getValue(); + inOrder.verify(mockSubchannel3).requestConnection(); + when(mockSubchannel3.getAllAddresses()).thenReturn(Lists.newArrayList(servers.get(0))); + stateListener3.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + inOrder.verify(mockHelper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // first subchannel connection attempt fails + stateListener3.onSubchannelState(ConnectivityStateInfo.forTransientFailure(error)); + assertEquals(CONNECTING, loadBalancer.getCurrentState()); + + // second subchannel connection attempt + inOrder.verify(mockSubchannel2).requestConnection(); + stateListener2.onSubchannelState(ConnectivityStateInfo.forNonError(READY)); + assertEquals(READY, loadBalancer.getCurrentState()); + + // verify that picker returns correct subchannel + inOrder.verify(mockHelper).updateBalancingState(eq(READY), pickerCaptor.capture()); + inOrder.verify(mockSubchannel3).shutdown(); + picker = pickerCaptor.getValue(); + assertEquals(PickResult.withSubchannel(mockSubchannel2), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel1), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel3), picker.pickSubchannel(mockArgs)); + assertNotEquals(PickResult.withSubchannel(mockSubchannel4), picker.pickSubchannel(mockArgs)); + } + + + @Test + public void index_looping() { + Attributes.Key key = Attributes.Key.create("some-key"); + Attributes attr1 = Attributes.newBuilder().set(key, "1").build(); + Attributes attr2 = Attributes.newBuilder().set(key, "2").build(); + Attributes attr3 = Attributes.newBuilder().set(key, "3").build(); + SocketAddress addr1 = new FakeSocketAddress("addr1"); + SocketAddress addr2 = new FakeSocketAddress("addr2"); + SocketAddress addr3 = new FakeSocketAddress("addr3"); + SocketAddress addr4 = new FakeSocketAddress("addr4"); + SocketAddress addr5 = new FakeSocketAddress("addr5"); + PickFirstLeafLoadBalancer.Index index = new PickFirstLeafLoadBalancer.Index(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1, addr2), attr1), + new EquivalentAddressGroup(Arrays.asList(addr3), attr2), + new EquivalentAddressGroup(Arrays.asList(addr4, addr5), attr3))); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + assertThat(index.isAtBeginning()).isTrue(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr2); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr3); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr2); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr4); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr3); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr5); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr3); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isTrue(); + + index.increment(); + assertThat(index.isAtBeginning()).isFalse(); + assertThat(index.isValid()).isFalse(); + + index.reset(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + assertThat(index.isAtBeginning()).isTrue(); + assertThat(index.isValid()).isTrue(); + + // We want to make sure both groupIndex and addressIndex are reset + index.increment(); + index.increment(); + index.increment(); + index.increment(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr5); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr3); + index.reset(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.getCurrentEagAttributes()).isSameInstanceAs(attr1); + } + + @Test + public void index_updateGroups_resets() { + SocketAddress addr1 = new FakeSocketAddress("addr1"); + SocketAddress addr2 = new FakeSocketAddress("addr2"); + SocketAddress addr3 = new FakeSocketAddress("addr3"); + PickFirstLeafLoadBalancer.Index index = new PickFirstLeafLoadBalancer.Index(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1)), + new EquivalentAddressGroup(Arrays.asList(addr2, addr3)))); + index.increment(); + index.increment(); + // We want to make sure both groupIndex and addressIndex are reset + index.updateGroups(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1)), + new EquivalentAddressGroup(Arrays.asList(addr2, addr3)))); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + } + + @Test + public void index_seekTo() { + SocketAddress addr1 = new FakeSocketAddress("addr1"); + SocketAddress addr2 = new FakeSocketAddress("addr2"); + SocketAddress addr3 = new FakeSocketAddress("addr3"); + PickFirstLeafLoadBalancer.Index index = new PickFirstLeafLoadBalancer.Index(Arrays.asList( + new EquivalentAddressGroup(Arrays.asList(addr1, addr2)), + new EquivalentAddressGroup(Arrays.asList(addr3)))); + assertThat(index.seekTo(addr3)).isTrue(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr3); + assertThat(index.seekTo(addr1)).isTrue(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr1); + assertThat(index.seekTo(addr2)).isTrue(); + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr2); + index.seekTo(new FakeSocketAddress("addr4")); + // Failed seekTo doesn't change the index + assertThat(index.getCurrentAddress()).isSameInstanceAs(addr2); + } + + private static class FakeSocketAddress extends SocketAddress { + final String name; + + FakeSocketAddress(String name) { + this.name = name; + } + + @Override + public String toString() { + return "FakeSocketAddress-" + name; + } + + } + + private static class FakeSubchannel extends Subchannel { + private final Attributes attributes; + private List eags; + private SubchannelStateListener listener; + + public FakeSubchannel(List eags, Attributes attributes) { + this.eags = Collections.unmodifiableList(eags); + this.attributes = attributes; + } + + @Override + public List getAllAddresses() { + return eags; + } + + @Override + public Attributes getAttributes() { + return attributes; + } + + @Override + public void start(SubchannelStateListener listener) { + this.listener = checkNotNull(listener, "listener"); + } + + @Override + public void updateAddresses(List addrs) { + this.eags = Collections.unmodifiableList(addrs); + } + + @Override + public void shutdown() { + } + + @Override + public void requestConnection() { + listener.onSubchannelState(ConnectivityStateInfo.forNonError(CONNECTING)); + } + } +} \ No newline at end of file diff --git a/examples/README.md b/examples/README.md index ae849fa7d6d..9664942b5fa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -205,6 +205,8 @@ $ bazel-bin/hello-world-client - [JWT-based Authentication](example-jwt-auth) +- [Pre-serialized messages](src/main/java/io/grpc/examples/preserialized) + ## Unit test examples Examples for unit testing gRPC clients and servers are located in [examples/src/test](src/test). diff --git a/examples/android/clientcache/app/build.gradle b/examples/android/clientcache/app/build.gradle index 1a481266956..e2fead073a0 100644 --- a/examples/android/clientcache/app/build.gradle +++ b/examples/android/clientcache/app/build.gradle @@ -32,7 +32,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index 4cc2100d23c..0ec3da6410c 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -30,7 +30,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index acadd301b51..92af8f3706d 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -30,7 +30,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/strictmode/app/build.gradle b/examples/android/strictmode/app/build.gradle index e8eb82e1373..031140f16ce 100644 --- a/examples/android/strictmode/app/build.gradle +++ b/examples/android/strictmode/app/build.gradle @@ -31,7 +31,7 @@ android { } protobuf { - protoc { artifact = 'com.google.protobuf:protoc:3.23.4' } + protoc { artifact = 'com.google.protobuf:protoc:3.24.0' } plugins { grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/build.gradle b/examples/build.gradle index 79ac3074aaf..7322ae29082 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -24,7 +24,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' def protocVersion = protobufVersion dependencies { @@ -110,6 +110,8 @@ createStartScripts('io.grpc.examples.multiplex.MultiplexingServer') createStartScripts('io.grpc.examples.multiplex.SharingClient') createStartScripts('io.grpc.examples.nameresolve.NameResolveClient') createStartScripts('io.grpc.examples.nameresolve.NameResolveServer') +createStartScripts('io.grpc.examples.preserialized.PreSerializedClient') +createStartScripts('io.grpc.examples.preserialized.PreSerializedServer') createStartScripts('io.grpc.examples.retrying.RetryingHelloWorldClient') createStartScripts('io.grpc.examples.retrying.RetryingHelloWorldServer') createStartScripts('io.grpc.examples.routeguide.RouteGuideClient') diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index 7830538a766..953e53a829c 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { // grpc-alts transitively depends on grpc-netty-shaded, grpc-protobuf, and grpc-stub diff --git a/examples/example-debug/build.gradle b/examples/example-debug/build.gradle index 84463a0ed44..90922ab1f5e 100644 --- a/examples/example-debug/build.gradle +++ b/examples/example-debug/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-debug/pom.xml b/examples/example-debug/pom.xml index 95ac88c813f..71d085d1559 100644 --- a/examples/example-debug/pom.xml +++ b/examples/example-debug/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index c2b23d41a90..2f1a0a8a3b1 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' def protocVersion = protobufVersion diff --git a/examples/example-gauth/pom.xml b/examples/example-gauth/pom.xml index 3edbf030179..a92db60b1a9 100644 --- a/examples/example-gauth/pom.xml +++ b/examples/example-gauth/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-gcp-observability/build.gradle b/examples/example-gcp-observability/build.gradle index 67242246305..9df847277dd 100644 --- a/examples/example-gcp-observability/build.gradle +++ b/examples/example-gcp-observability/build.gradle @@ -27,7 +27,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-hostname/build.gradle b/examples/example-hostname/build.gradle index bd60e8b77d2..e940e756801 100644 --- a/examples/example-hostname/build.gradle +++ b/examples/example-hostname/build.gradle @@ -24,7 +24,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-hostname/pom.xml b/examples/example-hostname/pom.xml index 1cec5319423..422bc6a554c 100644 --- a/examples/example-hostname/pom.xml +++ b/examples/example-hostname/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index d6ebdb0f161..3964e0a6276 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -25,7 +25,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protobufVersion = '3.23.4' +def protobufVersion = '3.24.0' def protocVersion = protobufVersion dependencies { diff --git a/examples/example-jwt-auth/pom.xml b/examples/example-jwt-auth/pom.xml index 0ed7b4cc032..d955518a8cf 100644 --- a/examples/example-jwt-auth/pom.xml +++ b/examples/example-jwt-auth/pom.xml @@ -14,8 +14,8 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 - 3.23.4 + 3.24.0 + 3.24.0 1.8 1.8 diff --git a/examples/example-orca/build.gradle b/examples/example-orca/build.gradle index 852403fe240..c32e5ca03b5 100644 --- a/examples/example-orca/build.gradle +++ b/examples/example-orca/build.gradle @@ -20,7 +20,7 @@ java { } def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-reflection/build.gradle b/examples/example-reflection/build.gradle index c4499953827..2b9971981d5 100644 --- a/examples/example-reflection/build.gradle +++ b/examples/example-reflection/build.gradle @@ -20,7 +20,7 @@ java { } def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-servlet/build.gradle b/examples/example-servlet/build.gradle index 1563a88633a..54e65ead77e 100644 --- a/examples/example-servlet/build.gradle +++ b/examples/example-servlet/build.gradle @@ -18,7 +18,7 @@ java { } def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}", diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index ffa42d8bfe2..1144bf04c6c 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -26,7 +26,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/example-tls/pom.xml b/examples/example-tls/pom.xml index 371d3bce03f..cd0cb87d2a2 100644 --- a/examples/example-tls/pom.xml +++ b/examples/example-tls/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 + 3.24.0 1.8 1.8 diff --git a/examples/example-xds/build.gradle b/examples/example-xds/build.gradle index 9222ce2d7fc..74175c2ba07 100644 --- a/examples/example-xds/build.gradle +++ b/examples/example-xds/build.gradle @@ -25,7 +25,7 @@ java { // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. def grpcVersion = '1.58.0-SNAPSHOT' // CURRENT_GRPC_VERSION -def protocVersion = '3.23.4' +def protocVersion = '3.24.0' dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" diff --git a/examples/pom.xml b/examples/pom.xml index 793604cd804..3563d599851 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -13,8 +13,8 @@ UTF-8 1.58.0-SNAPSHOT - 3.23.4 - 3.23.4 + 3.24.0 + 3.24.0 1.8 1.8 diff --git a/examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java b/examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java new file mode 100644 index 00000000000..c6f099280f9 --- /dev/null +++ b/examples/src/main/java/io/grpc/examples/preserialized/ByteArrayMarshaller.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.examples.preserialized; + +import com.google.common.io.ByteStreams; +import io.grpc.MethodDescriptor; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * A marshaller that produces a byte[] instead of decoding into typical POJOs. It can be used for + * any message type. + */ +final class ByteArrayMarshaller implements MethodDescriptor.Marshaller { + @Override + public byte[] parse(InputStream stream) { + try { + return ByteStreams.toByteArray(stream); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public InputStream stream(byte[] b) { + return new ByteArrayInputStream(b); + } +} diff --git a/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java new file mode 100644 index 00000000000..511e8a177f8 --- /dev/null +++ b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedClient.java @@ -0,0 +1,108 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.examples.preserialized; + +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.Grpc; +import io.grpc.InsecureChannelCredentials; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; +import io.grpc.examples.helloworld.GreeterGrpc; +import io.grpc.examples.helloworld.HelloReply; +import io.grpc.examples.helloworld.HelloRequest; +import io.grpc.stub.ClientCalls; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A client that requests a greeting from a hello-world server, but using a pre-serialized request. + * This is a performance optimization that can be useful if you read the request from on-disk or a + * database where it is already serialized, or if you need to send the same complicated message to + * many servers. The same approach can avoid deserializing responses, to be stored in a database. + * This adjustment is client-side only; the server is unable to detect the difference, so this + * client is fully-compatible with the normal {@link HelloWorldServer}. + */ +public class PreSerializedClient { + private static final Logger logger = Logger.getLogger(PreSerializedClient.class.getName()); + + /** + * Modified sayHello() descriptor with bytes as the request, instead of HelloRequest. By adjusting + * toBuilder() you can choose which of the request and response are bytes. + */ + private static final MethodDescriptor SAY_HELLO + = GreeterGrpc.getSayHelloMethod() + .toBuilder(new ByteArrayMarshaller(), GreeterGrpc.getSayHelloMethod().getResponseMarshaller()) + .build(); + + private final Channel channel; + + /** Construct client for accessing hello-world server using the existing channel. */ + public PreSerializedClient(Channel channel) { + this.channel = channel; + } + + /** Say hello to server. */ + public void greet(String name) { + logger.info("Will try to greet " + name + " ..."); + byte[] request = HelloRequest.newBuilder().setName(name).build().toByteArray(); + HelloReply response; + try { + // Stubs use ClientCalls to send RPCs. Since the generated stub won't have byte[] in its + // method signature, this uses ClientCalls directly. It isn't as convenient, but it behaves + // the same as a normal stub. + response = ClientCalls.blockingUnaryCall(channel, SAY_HELLO, CallOptions.DEFAULT, request); + } catch (StatusRuntimeException e) { + logger.log(Level.WARNING, "RPC failed: {0}", e.getStatus()); + return; + } + logger.info("Greeting: " + response.getMessage()); + } + + /** + * Greet server. If provided, the first element of {@code args} is the name to use in the + * greeting. The second argument is the target server. + */ + public static void main(String[] args) throws Exception { + String user = "world"; + String target = "localhost:50051"; + if (args.length > 0) { + if ("--help".equals(args[0])) { + System.err.println("Usage: [name [target]]"); + System.err.println(""); + System.err.println(" name The name you wish to be greeted by. Defaults to " + user); + System.err.println(" target The server to connect to. Defaults to " + target); + System.exit(1); + } + user = args[0]; + } + if (args.length > 1) { + target = args[1]; + } + + ManagedChannel channel = Grpc.newChannelBuilder(target, InsecureChannelCredentials.create()) + .build(); + try { + PreSerializedClient client = new PreSerializedClient(channel); + client.greet(user); + } finally { + channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + } + } +} diff --git a/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java new file mode 100644 index 00000000000..51beca57386 --- /dev/null +++ b/examples/src/main/java/io/grpc/examples/preserialized/PreSerializedServer.java @@ -0,0 +1,164 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.examples.preserialized; + +import io.grpc.BindableService; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; +import io.grpc.MethodDescriptor; +import io.grpc.Server; +import io.grpc.ServerCallHandler; +import io.grpc.ServerMethodDefinition; +import io.grpc.ServerServiceDefinition; +import io.grpc.ServiceDescriptor; +import io.grpc.examples.helloworld.GreeterGrpc; +import io.grpc.examples.helloworld.HelloReply; +import io.grpc.examples.helloworld.HelloRequest; +import io.grpc.stub.ServerCalls; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Server that provides a {@code Greeter} service, but that uses a pre-serialized response. This is + * a performance optimization that can be useful if you read the response from on-disk or a database + * where it is already serialized, or if you need to send the same complicated message to many + * clients. The same approach can avoid deserializing requests, to be stored in a database. This + * adjustment is server-side only; the client is unable to detect the differences, so this server is + * fully-compatible with the normal {@link HelloWorldClient}. + */ +public class PreSerializedServer { + private static final Logger logger = Logger.getLogger(PreSerializedServer.class.getName()); + + private Server server; + + private void start() throws IOException { + int port = 50051; + server = Grpc.newServerBuilderForPort(port, InsecureServerCredentials.create()) + .addService(new GreeterImpl()) + .build() + .start(); + logger.info("Server started, listening on " + port); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + // Use stderr here since the logger may have been reset by its JVM shutdown hook. + System.err.println("*** shutting down gRPC server since JVM is shutting down"); + try { + PreSerializedServer.this.stop(); + } catch (InterruptedException e) { + e.printStackTrace(System.err); + } + System.err.println("*** server shut down"); + } + }); + } + + private void stop() throws InterruptedException { + if (server != null) { + server.shutdown().awaitTermination(30, TimeUnit.SECONDS); + } + } + + /** + * Await termination on the main thread since the grpc library uses daemon threads. + */ + private void blockUntilShutdown() throws InterruptedException { + if (server != null) { + server.awaitTermination(); + } + } + + /** + * Main launches the server from the command line. + */ + public static void main(String[] args) throws IOException, InterruptedException { + final PreSerializedServer server = new PreSerializedServer(); + server.start(); + server.blockUntilShutdown(); + } + + static class GreeterImpl implements GreeterGrpc.AsyncService, BindableService { + + public void byteSayHello(HelloRequest req, StreamObserver responseObserver) { + HelloReply reply = HelloReply.newBuilder().setMessage("Hello " + req.getName()).build(); + responseObserver.onNext(reply.toByteArray()); + responseObserver.onCompleted(); + } + + @Override + public ServerServiceDefinition bindService() { + MethodDescriptor sayHello = GreeterGrpc.getSayHelloMethod(); + // Modifying the method descriptor to use bytes as the response, instead of HelloReply. By + // adjusting toBuilder() you can choose which of the request and response are bytes. + MethodDescriptor byteSayHello = sayHello + .toBuilder(sayHello.getRequestMarshaller(), new ByteArrayMarshaller()) + .build(); + // GreeterGrpc.bindService() will bind every service method, including sayHello(). (Although + // Greeter only has one method, this approach would work for any service.) AsyncService + // provides a default implementation of sayHello() that returns UNIMPLEMENTED, and that + // implementation will be used by bindService(). replaceMethod() will rewrite that method to + // use our byte-based method instead. + // + // The generated bindService() uses ServerCalls to make RPC handlers. Since the generated + // bindService() won't expect byte[] in the AsyncService, this uses ServerCalls directly. It + // isn't as convenient, but it behaves the same as a normal RPC handler. + return replaceMethod( + GreeterGrpc.bindService(this), + byteSayHello, + ServerCalls.asyncUnaryCall(this::byteSayHello)); + } + + /** Rewrites the ServerServiceDefinition replacing one method's definition. */ + private static ServerServiceDefinition replaceMethod( + ServerServiceDefinition def, + MethodDescriptor newDesc, + ServerCallHandler newHandler) { + // There are two data structures involved. The first is the "descriptor" which describes the + // service and methods as a schema. This is the same on client and server. The second is the + // "definition" which includes the handlers to execute methods. This is specific to the server + // and is generated by "bind." This adjusts both the descriptor and definition. + + // Descriptor + ServiceDescriptor desc = def.getServiceDescriptor(); + ServiceDescriptor.Builder descBuilder = ServiceDescriptor.newBuilder(desc.getName()) + .setSchemaDescriptor(desc.getSchemaDescriptor()) + .addMethod(newDesc); // Add the modified method + // Copy methods other than the modified one + for (MethodDescriptor md : desc.getMethods()) { + if (newDesc.getFullMethodName().equals(md.getFullMethodName())) { + continue; + } + descBuilder.addMethod(md); + } + + // Definition + ServerServiceDefinition.Builder defBuilder = + ServerServiceDefinition.builder(descBuilder.build()) + .addMethod(newDesc, newHandler); // Add the modified method + // Copy methods other than the modified one + for (ServerMethodDefinition smd : def.getMethods()) { + if (newDesc.getFullMethodName().equals(smd.getMethodDescriptor().getFullMethodName())) { + continue; + } + defBuilder.addMethod(smd); + } + return defBuilder.build(); + } + } +} diff --git a/gae-interop-testing/gae-jdk8/build.gradle b/gae-interop-testing/gae-jdk8/build.gradle index f3ff765ddfb..81aeda54a49 100644 --- a/gae-interop-testing/gae-jdk8/build.gradle +++ b/gae-interop-testing/gae-jdk8/build.gradle @@ -166,3 +166,7 @@ tasks.register("runInteropTestRemote") { throw new GradleException("Interop test failed:\nthrowable:${caught}") } } + +tasks.named("javadoc").configure { + enabled = false +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5a248478f08..9ed986c9aa2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ netty = '4.1.93.Final' # SECURITY.md nettytcnative = '2.0.61.Final' opencensus = "0.31.1" -protobuf = "3.23.4" +protobuf = "3.24.0" [libraries] android-annotations = "com.google.android:annotations:4.1.1.4" diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java index 82c49b5813b..48001cbdc1f 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java @@ -2007,18 +2007,21 @@ public Status getStatus() { } private SoakIterationResult performOneSoakIteration( - TestServiceGrpc.TestServiceBlockingStub soakStub) throws Exception { + TestServiceGrpc.TestServiceBlockingStub soakStub, int soakRequestSize, int soakResponseSize) + throws Exception { long startNs = System.nanoTime(); Status status = Status.OK; try { final SimpleRequest request = SimpleRequest.newBuilder() - .setResponseSize(314159) - .setPayload(Payload.newBuilder().setBody(ByteString.copyFrom(new byte[271828]))) + .setResponseSize(soakResponseSize) + .setPayload( + Payload.newBuilder().setBody(ByteString.copyFrom(new byte[soakRequestSize]))) .build(); final SimpleResponse goldenResponse = SimpleResponse.newBuilder() - .setPayload(Payload.newBuilder().setBody(ByteString.copyFrom(new byte[314159]))) + .setPayload( + Payload.newBuilder().setBody(ByteString.copyFrom(new byte[soakResponseSize]))) .build(); assertResponse(goldenResponse, soakStub.unaryCall(request)); } catch (StatusRuntimeException e) { @@ -2039,7 +2042,9 @@ public void performSoakTest( int maxFailures, int maxAcceptablePerIterationLatencyMs, int minTimeMsBetweenRpcs, - int overallTimeoutSeconds) + int overallTimeoutSeconds, + int soakRequestSize, + int soakResponseSize) throws Exception { int iterationsDone = 0; int totalFailures = 0; @@ -2063,7 +2068,8 @@ public void performSoakTest( .newBlockingStub(soakChannel) .withInterceptors(recordClientCallInterceptor(clientCallCapture)); } - SoakIterationResult result = performOneSoakIteration(soakStub); + SoakIterationResult result = + performOneSoakIteration(soakStub, soakRequestSize, soakResponseSize); SocketAddress peer = clientCallCapture .get().getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); StringBuilder logStr = new StringBuilder( diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java index a6e2c4f3bf8..768a32be62c 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java @@ -97,6 +97,8 @@ public static void main(String[] args) throws Exception { private int soakMinTimeMsBetweenRpcs = 0; private int soakOverallTimeoutSeconds = soakIterations * soakPerIterationMaxAcceptableLatencyMs / 1000; + private int soakRequestSize = 271828; + private int soakResponseSize = 314159; private String additionalMetadata = ""; private static LoadBalancerProvider customBackendMetricsLoadBalancerProvider; @@ -175,6 +177,10 @@ void parseArgs(String[] args) throws Exception { soakMinTimeMsBetweenRpcs = Integer.parseInt(value); } else if ("soak_overall_timeout_seconds".equals(key)) { soakOverallTimeoutSeconds = Integer.parseInt(value); + } else if ("soak_request_size".equals(key)) { + soakRequestSize = Integer.parseInt(value); + } else if ("soak_response_size".equals(key)) { + soakResponseSize = Integer.parseInt(value); } else if ("additional_metadata".equals(key)) { additionalMetadata = value; } else { @@ -247,6 +253,12 @@ void parseArgs(String[] args) throws Exception { + "\n should stop and fail, if the desired number of " + "\n iterations have not yet completed. Default " + c.soakOverallTimeoutSeconds + + "\n --soak_request_size " + + "\n The request size in a soak RPC. Default " + + c.soakRequestSize + + "\n --soak_response_size " + + "\n The response size in a soak RPC. Default " + + c.soakResponseSize + "\n --additional_metadata " + "\n Additional metadata to send in each request, as a " + "\n semicolon-separated list of key:value pairs. Default " @@ -481,7 +493,9 @@ private void runTest(TestCases testCase) throws Exception { soakMaxFailures, soakPerIterationMaxAcceptableLatencyMs, soakMinTimeMsBetweenRpcs, - soakOverallTimeoutSeconds); + soakOverallTimeoutSeconds, + soakRequestSize, + soakResponseSize); break; } @@ -493,7 +507,9 @@ private void runTest(TestCases testCase) throws Exception { soakMaxFailures, soakPerIterationMaxAcceptableLatencyMs, soakMinTimeMsBetweenRpcs, - soakOverallTimeoutSeconds); + soakOverallTimeoutSeconds, + soakRequestSize, + soakResponseSize); break; } diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java index 8f166b6affa..9b01df0d18d 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/XdsFederationTestClient.java @@ -74,6 +74,8 @@ public void run() { private int soakPerIterationMaxAcceptableLatencyMs = 1000; private int soakOverallTimeoutSeconds = 10; private int soakMinTimeMsBetweenRpcs = 0; + private int soakRequestSize = 271828; + private int soakResponseSize = 314159; private String testCase = "rpc_soak"; private final ArrayList clients = new ArrayList<>(); @@ -122,6 +124,12 @@ private void parseArgs(String[] args) { case "soak_min_time_ms_between_rpcs": soakMinTimeMsBetweenRpcs = Integer.parseInt(value); break; + case "soak_request_size": + soakRequestSize = Integer.parseInt(value); + break; + case "soak_response_size": + soakResponseSize = Integer.parseInt(value); + break; default: System.err.println("Unknown argument: " + key); usage = true; @@ -175,6 +183,14 @@ private void parseArgs(String[] args) { + "\n channel_soak: sends --soak_iterations RPCs, rebuilding the channel " + "each time." + "\n Default: " + c.testCase + + "\n --soak_request_size " + + "\n The request size in a soak RPC. Default " + + c.soakRequestSize + + "\n" + + " --soak_response_size \n" + + " The response size in a soak RPC. Default" + + " " + + c.soakResponseSize ); System.exit(1); } @@ -249,7 +265,9 @@ public void run() { soakMaxFailures, soakPerIterationMaxAcceptableLatencyMs, soakMinTimeMsBetweenRpcs, - soakOverallTimeoutSeconds); + soakOverallTimeoutSeconds, + soakRequestSize, + soakResponseSize); logger.info("Test case: " + testCase + " done for server: " + serverUri); runSucceeded = true; } catch (Exception e) { diff --git a/java_grpc_library.bzl b/java_grpc_library.bzl index 4c7a7f2791b..88ad8b889a3 100644 --- a/java_grpc_library.bzl +++ b/java_grpc_library.bzl @@ -3,6 +3,7 @@ _JavaRpcToolchainInfo = provider( fields = [ "java_toolchain", + "java_plugins", "plugin", "plugin_arg", "protoc", @@ -14,6 +15,7 @@ def _java_rpc_toolchain_impl(ctx): return [ _JavaRpcToolchainInfo( java_toolchain = ctx.attr._java_toolchain, + java_plugins = ctx.attr.java_plugins, plugin = ctx.executable.plugin, plugin_arg = ctx.attr.plugin_arg, protoc = ctx.executable._protoc, @@ -39,6 +41,10 @@ java_rpc_toolchain = rule( default = Label("@com_google_protobuf//:protoc"), executable = True, ), + "java_plugins": attr.label_list( + default = [], + providers = [JavaPluginInfo], + ), "_java_toolchain": attr.label( default = Label("@bazel_tools//tools/jdk:current_java_toolchain"), ), @@ -104,6 +110,7 @@ def _java_rpc_library_impl(ctx): source_jars = [srcjar], output = ctx.outputs.jar, output_source_jar = ctx.outputs.srcjar, + plugins = [plugin[JavaPluginInfo] for plugin in toolchain.java_plugins], deps = [ java_common.make_non_strict(deps_java_info), ] + [dep[JavaInfo] for dep in toolchain.runtime], diff --git a/okhttp/build.gradle b/okhttp/build.gradle index 7d84df436db..99f799ec97f 100644 --- a/okhttp/build.gradle +++ b/okhttp/build.gradle @@ -50,6 +50,7 @@ tasks.named("checkstyleMain").configure { tasks.named("javadoc").configure { options.links 'http://square.github.io/okhttp/2.x/okhttp/' exclude 'io/grpc/okhttp/Internal*' + exclude 'io/grpc/okhttp/*Provider.java' exclude 'io/grpc/okhttp/internal/**' } diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java index 45d6b9efc54..8269a8ddf0f 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpServerBuilder.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.DoNotCall; @@ -89,7 +88,7 @@ public final class OkHttpServerBuilder extends ForwardingServerBuilder provider.builderForPort(80)); + } + + @Test + public void newServerBuilderForPort_success() { + ServerProvider.NewServerBuilderResult result = + provider.newServerBuilderForPort(80, InsecureServerCredentials.create()); + assertThat(result.getServerBuilder()).isInstanceOf(OkHttpServerBuilder.class); + } + + @Test + public void newServerBuilderForPort_fail() { + ServerProvider.NewServerBuilderResult result = provider.newServerBuilderForPort( + 80, new FakeServerCredentials()); + assertThat(result.getError()).contains("FakeServerCredentials"); + } + + private static final class FakeServerCredentials extends ServerCredentials {} +} diff --git a/repositories.bzl b/repositories.bzl index ce86fbd68c0..d9a383a3e89 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -143,18 +143,18 @@ def com_google_protobuf(): # This statement defines the @com_google_protobuf repo. http_archive( name = "com_google_protobuf", - sha256 = "ac3fd4e97af55405d8bfba43c22d8a7e464a371bb6bc9e706627b745c1022dbf", - strip_prefix = "protobuf-23.4", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protobuf-23.4.zip"], + sha256 = "5980276108f948e1ada091475549a8c75dc83c193129aab0e986ceaac3e97131", + strip_prefix = "protobuf-24.0", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protobuf-24.0.zip"], ) def com_google_protobuf_javalite(): # java_lite_proto_library rules implicitly depend on @com_google_protobuf_javalite http_archive( name = "com_google_protobuf_javalite", - sha256 = "ac3fd4e97af55405d8bfba43c22d8a7e464a371bb6bc9e706627b745c1022dbf", - strip_prefix = "protobuf-23.4", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protobuf-23.4.zip"], + sha256 = "5980276108f948e1ada091475549a8c75dc83c193129aab0e986ceaac3e97131", + strip_prefix = "protobuf-24.0", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protobuf-24.0.zip"], ) def io_grpc_grpc_proto(): diff --git a/util/build.gradle b/util/build.gradle index af4e6044ef2..6234bdc8f86 100644 --- a/util/build.gradle +++ b/util/build.gradle @@ -37,3 +37,7 @@ animalsniffer { sourceSets.test ] } + +tasks.named("javadoc").configure { + exclude 'io/grpc/util/MultiChildLoadBalancer.java' +} diff --git a/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java b/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java new file mode 100644 index 00000000000..be0a23a1642 --- /dev/null +++ b/util/src/main/java/io/grpc/util/MultiChildLoadBalancer.java @@ -0,0 +1,264 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.util; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.IDLE; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; + +import com.google.common.annotations.VisibleForTesting; +import io.grpc.ConnectivityState; +import io.grpc.Internal; +import io.grpc.LoadBalancer; +import io.grpc.LoadBalancerProvider; +import io.grpc.Status; +import io.grpc.SynchronizationContext; +import io.grpc.SynchronizationContext.ScheduledHandle; +import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * A base load balancing policy for those policies which has multiple children such as + * ClusterManager or the petiole policies. For internal use only. + */ +@Internal +public abstract class MultiChildLoadBalancer extends LoadBalancer { + + @VisibleForTesting + public static final int DELAYED_CHILD_DELETION_TIME_MINUTES = 15; + private static final Logger logger = Logger.getLogger(MultiChildLoadBalancer.class.getName()); + private final Map childLbStates = new HashMap<>(); + private final Helper helper; + protected final SynchronizationContext syncContext; + private final ScheduledExecutorService timeService; + // Set to true if currently in the process of handling resolved addresses. + private boolean resolvingAddresses; + + protected MultiChildLoadBalancer(Helper helper) { + this.helper = checkNotNull(helper, "helper"); + this.syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext"); + this.timeService = checkNotNull(helper.getScheduledExecutorService(), "timeService"); + logger.log(Level.FINE, "Created"); + } + + protected SubchannelPicker getInitialPicker() { + return EMPTY_PICKER; + } + + protected SubchannelPicker getErrorPicker(Status error) { + return new ErrorPicker(error); + } + + protected abstract Map getPolicySelectionMap( + ResolvedAddresses resolvedAddresses); + + protected abstract SubchannelPicker getSubchannelPicker( + Map childPickers); + + @Override + public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { + try { + resolvingAddresses = true; + return acceptResolvedAddressesInternal(resolvedAddresses); + } finally { + resolvingAddresses = false; + } + } + + private boolean acceptResolvedAddressesInternal(ResolvedAddresses resolvedAddresses) { + logger.log(Level.FINE, "Received resolution result: {0}", resolvedAddresses); + Map newChildPolicies = getPolicySelectionMap(resolvedAddresses); + for (Map.Entry entry : newChildPolicies.entrySet()) { + final Object key = entry.getKey(); + LoadBalancerProvider childPolicyProvider = entry.getValue().getProvider(); + Object childConfig = entry.getValue().getConfig(); + if (!childLbStates.containsKey(key)) { + childLbStates.put(key, new ChildLbState(key, childPolicyProvider, getInitialPicker())); + } else { + childLbStates.get(key).reactivate(childPolicyProvider); + } + LoadBalancer childLb = childLbStates.get(key).lb; + ResolvedAddresses childAddresses = + resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(childConfig).build(); + childLb.handleResolvedAddresses(childAddresses); + } + for (Object key : childLbStates.keySet()) { + if (!newChildPolicies.containsKey(key)) { + childLbStates.get(key).deactivate(); + } + } + // Must update channel picker before return so that new RPCs will not be routed to deleted + // clusters and resolver can remove them in service config. + updateOverallBalancingState(); + return true; + } + + @Override + public void handleNameResolutionError(Status error) { + logger.log(Level.WARNING, "Received name resolution error: {0}", error); + boolean gotoTransientFailure = true; + for (ChildLbState state : childLbStates.values()) { + if (!state.deactivated) { + gotoTransientFailure = false; + state.lb.handleNameResolutionError(error); + } + } + if (gotoTransientFailure) { + helper.updateBalancingState(TRANSIENT_FAILURE, getErrorPicker(error)); + } + } + + @Override + public void shutdown() { + logger.log(Level.INFO, "Shutdown"); + for (ChildLbState state : childLbStates.values()) { + state.shutdown(); + } + childLbStates.clear(); + } + + private void updateOverallBalancingState() { + ConnectivityState overallState = null; + final Map childPickers = new HashMap<>(); + for (ChildLbState childLbState : childLbStates.values()) { + if (childLbState.deactivated) { + continue; + } + childPickers.put(childLbState.key, childLbState.currentPicker); + overallState = aggregateState(overallState, childLbState.currentState); + } + if (overallState != null) { + helper.updateBalancingState(overallState, getSubchannelPicker(childPickers)); + } + } + + @Nullable + private static ConnectivityState aggregateState( + @Nullable ConnectivityState overallState, ConnectivityState childState) { + if (overallState == null) { + return childState; + } + if (overallState == READY || childState == READY) { + return READY; + } + if (overallState == CONNECTING || childState == CONNECTING) { + return CONNECTING; + } + if (overallState == IDLE || childState == IDLE) { + return IDLE; + } + return overallState; + } + + private final class ChildLbState { + private final Object key; + private final GracefulSwitchLoadBalancer lb; + private LoadBalancerProvider policyProvider; + private ConnectivityState currentState = CONNECTING; + private SubchannelPicker currentPicker; + private boolean deactivated; + @Nullable + ScheduledHandle deletionTimer; + + ChildLbState(Object key, LoadBalancerProvider policyProvider, SubchannelPicker initialPicker) { + this.key = key; + this.policyProvider = policyProvider; + lb = new GracefulSwitchLoadBalancer(new ChildLbStateHelper()); + lb.switchTo(policyProvider); + currentPicker = initialPicker; + } + + void deactivate() { + if (deactivated) { + return; + } + + class DeletionTask implements Runnable { + @Override + public void run() { + shutdown(); + childLbStates.remove(key); + } + } + + deletionTimer = + syncContext.schedule( + new DeletionTask(), + DELAYED_CHILD_DELETION_TIME_MINUTES, + TimeUnit.MINUTES, + timeService); + deactivated = true; + logger.log(Level.FINE, "Child balancer {0} deactivated", key); + } + + void reactivate(LoadBalancerProvider policyProvider) { + if (deletionTimer != null && deletionTimer.isPending()) { + deletionTimer.cancel(); + deactivated = false; + logger.log(Level.FINE, "Child balancer {0} reactivated", key); + } + if (!this.policyProvider.getPolicyName().equals(policyProvider.getPolicyName())) { + Object[] objects = { + key, this.policyProvider.getPolicyName(),policyProvider.getPolicyName()}; + logger.log(Level.FINE, "Child balancer {0} switching policy from {1} to {2}", objects); + lb.switchTo(policyProvider); + this.policyProvider = policyProvider; + } + } + + void shutdown() { + if (deletionTimer != null && deletionTimer.isPending()) { + deletionTimer.cancel(); + } + lb.shutdown(); + logger.log(Level.FINE, "Child balancer {0} deleted", key); + } + + private final class ChildLbStateHelper extends ForwardingLoadBalancerHelper { + + @Override + public void updateBalancingState(final ConnectivityState newState, + final SubchannelPicker newPicker) { + // If we are already in the process of resolving addresses, the overall balancing state + // will be updated at the end of it, and we don't need to trigger that update here. + if (!childLbStates.containsKey(key)) { + return; + } + // Subchannel picker and state are saved, but will only be propagated to the channel + // when the child instance exits deactivated state. + currentState = newState; + currentPicker = newPicker; + if (!deactivated && !resolvingAddresses) { + updateOverallBalancingState(); + } + } + + @Override + protected Helper delegate() { + return helper; + } + } + } +} diff --git a/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java b/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java index a1a4b4be282..bd8825474fa 100644 --- a/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java +++ b/util/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java @@ -394,47 +394,55 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { Subchannel subchannel = pickResult.getSubchannel(); if (subchannel != null) { - return PickResult.withSubchannel(subchannel, - new ResultCountingClientStreamTracerFactory( - subchannel.getAttributes().get(ADDRESS_TRACKER_ATTR_KEY))); + return PickResult.withSubchannel(subchannel, new ResultCountingClientStreamTracerFactory( + subchannel.getAttributes().get(ADDRESS_TRACKER_ATTR_KEY), + pickResult.getStreamTracerFactory())); } return pickResult; } /** - * Builds instances of {@link ResultCountingClientStreamTracer}. + * Builds instances of a {@link ClientStreamTracer} that increments the call count in the + * tracker for each closed stream. */ class ResultCountingClientStreamTracerFactory extends ClientStreamTracer.Factory { private final AddressTracker tracker; - ResultCountingClientStreamTracerFactory(AddressTracker tracker) { - this.tracker = tracker; - } - - @Override - public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) { - return new ResultCountingClientStreamTracer(tracker); - } - } + @Nullable + private final ClientStreamTracer.Factory delegateFactory; - /** - * Counts the results (successful/unsuccessful) of a particular {@link - * OutlierDetectionSubchannel}s streams and increments the counter in the associated {@link - * AddressTracker}. - */ - class ResultCountingClientStreamTracer extends ClientStreamTracer { - - AddressTracker tracker; - - public ResultCountingClientStreamTracer(AddressTracker tracker) { + ResultCountingClientStreamTracerFactory(AddressTracker tracker, + @Nullable ClientStreamTracer.Factory delegateFactory) { this.tracker = tracker; + this.delegateFactory = delegateFactory; } @Override - public void streamClosed(Status status) { - tracker.incrementCallCount(status.isOk()); + public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) { + if (delegateFactory != null) { + ClientStreamTracer delegateTracer = delegateFactory.newClientStreamTracer(info, headers); + return new ForwardingClientStreamTracer() { + @Override + protected ClientStreamTracer delegate() { + return delegateTracer; + } + + @Override + public void streamClosed(Status status) { + tracker.incrementCallCount(status.isOk()); + delegate().streamClosed(status); + } + }; + } else { + return new ClientStreamTracer() { + @Override + public void streamClosed(Status status) { + tracker.incrementCallCount(status.isOk()); + } + }; + } } } } diff --git a/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java b/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java index 18f9bbf549f..13f13421a1e 100644 --- a/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java +++ b/util/src/test/java/io/grpc/util/OutlierDetectionLoadBalancerTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; @@ -46,6 +47,7 @@ import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancer.SubchannelStateListener; import io.grpc.LoadBalancerProvider; +import io.grpc.Metadata; import io.grpc.Status; import io.grpc.SynchronizationContext; import io.grpc.internal.FakeClock; @@ -96,6 +98,10 @@ public class OutlierDetectionLoadBalancerTest { private Helper mockHelper; @Mock private SocketAddress mockSocketAddress; + @Mock + private ClientStreamTracer.Factory mockStreamTracerFactory; + @Mock + private ClientStreamTracer mockStreamTracer; @Captor private ArgumentCaptor connectivityStateCaptor; @@ -193,6 +199,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable { } }); + when(mockStreamTracerFactory.newClientStreamTracer(any(), + any())).thenReturn(mockStreamTracer); + loadBalancer = new OutlierDetectionLoadBalancer(mockHelper, fakeClock.getTimeProvider()); } @@ -355,6 +364,72 @@ public void delegatePick() throws Exception { readySubchannel); } + /** + * Any ClientStreamTracer.Factory set by the delegate picker should still get used. + */ + @Test + public void delegatePickTracerFactoryPreserved() throws Exception { + OutlierDetectionLoadBalancerConfig config = new OutlierDetectionLoadBalancerConfig.Builder() + .setSuccessRateEjection(new SuccessRateEjection.Builder().build()) + .setChildPolicy(new PolicySelection(fakeLbProvider, null)).build(); + + loadBalancer.acceptResolvedAddresses(buildResolvedAddress(config, servers.get(0))); + + // Make one of the subchannels READY. + final Subchannel readySubchannel = subchannels.values().iterator().next(); + deliverSubchannelState(readySubchannel, ConnectivityStateInfo.forNonError(READY)); + + verify(mockHelper, times(2)).updateBalancingState(stateCaptor.capture(), + pickerCaptor.capture()); + + // Make sure that we can pick the single READY subchannel. + SubchannelPicker picker = pickerCaptor.getAllValues().get(1); + PickResult pickResult = picker.pickSubchannel(mock(PickSubchannelArgs.class)); + + // Calls to a stream tracer created with the factory in the result should make it to a stream + // tracer the underlying LB/picker is using. + ClientStreamTracer clientStreamTracer = pickResult.getStreamTracerFactory() + .newClientStreamTracer(ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + clientStreamTracer.inboundHeaders(); + // The underlying fake LB provider is configured with a factory that returns a mock stream + // tracer. + verify(mockStreamTracer).inboundHeaders(); + } + + /** + * Assure the tracer works even when the underlying LB does not have a tracer to delegate to. + */ + @Test + public void delegatePickTracerFactoryNotSet() throws Exception { + // We set the mock factory to null to indicate that the delegate does not have its own tracer. + mockStreamTracerFactory = null; + + OutlierDetectionLoadBalancerConfig config = new OutlierDetectionLoadBalancerConfig.Builder() + .setSuccessRateEjection(new SuccessRateEjection.Builder().build()) + .setChildPolicy(new PolicySelection(fakeLbProvider, null)).build(); + + loadBalancer.acceptResolvedAddresses(buildResolvedAddress(config, servers.get(0))); + + // Make one of the subchannels READY. + final Subchannel readySubchannel = subchannels.values().iterator().next(); + deliverSubchannelState(readySubchannel, ConnectivityStateInfo.forNonError(READY)); + + verify(mockHelper, times(2)).updateBalancingState(stateCaptor.capture(), + pickerCaptor.capture()); + + // Make sure that we can pick the single READY subchannel. + SubchannelPicker picker = pickerCaptor.getAllValues().get(1); + PickResult pickResult = picker.pickSubchannel(mock(PickSubchannelArgs.class)); + + // With no delegate tracers factory a call to the OD tracer should still work + ClientStreamTracer clientStreamTracer = pickResult.getStreamTracerFactory() + .newClientStreamTracer(ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata()); + clientStreamTracer.inboundHeaders(); + + // Sanity check to make sure the delegate tracer does not get called. + verifyNoInteractions(mockStreamTracer); + } + /** * The success rate algorithm leaves a healthy set of addresses alone. */ @@ -1121,7 +1196,7 @@ void assertEjectedSubchannels(Set addresses) { } /** Round robin like fake load balancer. */ - private static final class FakeLoadBalancer extends LoadBalancer { + private final class FakeLoadBalancer extends LoadBalancer { private final Helper helper; List subchannelList; @@ -1159,7 +1234,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { if (lastPickIndex < 0 || lastPickIndex > subchannelList.size() - 1) { lastPickIndex = 0; } - return PickResult.withSubchannel(subchannelList.get(lastPickIndex++)); + return PickResult.withSubchannel(subchannelList.get(lastPickIndex++), + mockStreamTracerFactory); } }; helper.updateBalancingState(state, picker); diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java index a640bbd78b9..7257fdfd16b 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java @@ -39,7 +39,6 @@ import io.grpc.xds.XdsClusterResource.CdsUpdate; import io.grpc.xds.XdsClusterResource.CdsUpdate.ClusterType; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; diff --git a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java index b2be811d508..95ca1a33e15 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java @@ -17,7 +17,6 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -46,7 +45,6 @@ import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsNameResolverProvider.CallCounterProvider; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import io.grpc.xds.internal.security.SslContextProviderSupplier; import io.grpc.xds.orca.OrcaPerRequestUtil; import io.grpc.xds.orca.OrcaPerRequestUtil.OrcaPerRequestReportListener; @@ -173,7 +171,7 @@ public void shutdown() { private final class ClusterImplLbHelper extends ForwardingLoadBalancerHelper { private final AtomicLong inFlights; private ConnectivityState currentState = ConnectivityState.IDLE; - private SubchannelPicker currentPicker = BUFFER_PICKER; + private SubchannelPicker currentPicker = LoadBalancer.EMPTY_PICKER; private List dropPolicies = Collections.emptyList(); private long maxConcurrentRequests = DEFAULT_PER_CLUSTER_MAX_CONCURRENT_REQUESTS; @Nullable diff --git a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java index cce32c68246..a4489204236 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java @@ -16,266 +16,63 @@ package io.grpc.xds; -import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.IDLE; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; - -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import io.grpc.ConnectivityState; import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancerProvider; import io.grpc.Status; -import io.grpc.SynchronizationContext; -import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.util.GracefulSwitchLoadBalancer; +import io.grpc.util.MultiChildLoadBalancer; import io.grpc.xds.ClusterManagerLoadBalancerProvider.ClusterManagerConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; /** * The top-level load balancing policy. */ -class ClusterManagerLoadBalancer extends LoadBalancer { - - @VisibleForTesting - static final int DELAYED_CHILD_DELETION_TIME_MINUTES = 15; +class ClusterManagerLoadBalancer extends MultiChildLoadBalancer { - private final Map childLbStates = new HashMap<>(); - private final Helper helper; - private final SynchronizationContext syncContext; - private final ScheduledExecutorService timeService; private final XdsLogger logger; - // Set to true if currently in the process of handling resolved addresses. - private boolean resolvingAddresses; ClusterManagerLoadBalancer(Helper helper) { - this.helper = checkNotNull(helper, "helper"); - this.syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext"); - this.timeService = checkNotNull(helper.getScheduledExecutorService(), "timeService"); + super(helper); logger = XdsLogger.withLogId( InternalLogId.allocate("cluster_manager-lb", helper.getAuthority())); logger.log(XdsLogLevel.INFO, "Created"); } @Override - public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { - try { - resolvingAddresses = true; - return acceptResolvedAddressesInternal(resolvedAddresses); - } finally { - resolvingAddresses = false; - } - } - - public boolean acceptResolvedAddressesInternal(ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); + protected Map getPolicySelectionMap( + ResolvedAddresses resolvedAddresses) { ClusterManagerConfig config = (ClusterManagerConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); - Map newChildPolicies = config.childPolicies; + Map newChildPolicies = new HashMap<>(config.childPolicies); logger.log( XdsLogLevel.INFO, "Received cluster_manager lb config: child names={0}", newChildPolicies.keySet()); - for (Map.Entry entry : newChildPolicies.entrySet()) { - final String name = entry.getKey(); - LoadBalancerProvider childPolicyProvider = entry.getValue().getProvider(); - Object childConfig = entry.getValue().getConfig(); - if (!childLbStates.containsKey(name)) { - childLbStates.put(name, new ChildLbState(name, childPolicyProvider)); - } else { - childLbStates.get(name).reactivate(childPolicyProvider); - } - LoadBalancer childLb = childLbStates.get(name).lb; - ResolvedAddresses childAddresses = - resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(childConfig).build(); - childLb.handleResolvedAddresses(childAddresses); - } - for (String name : childLbStates.keySet()) { - if (!newChildPolicies.containsKey(name)) { - childLbStates.get(name).deactivate(); - } - } - // Must update channel picker before return so that new RPCs will not be routed to deleted - // clusters and resolver can remove them in service config. - updateOverallBalancingState(); - return true; - } - - @Override - public void handleNameResolutionError(Status error) { - logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - boolean gotoTransientFailure = true; - for (ChildLbState state : childLbStates.values()) { - if (!state.deactivated) { - gotoTransientFailure = false; - state.lb.handleNameResolutionError(error); - } - } - if (gotoTransientFailure) { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } + return newChildPolicies; } @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - for (ChildLbState state : childLbStates.values()) { - state.shutdown(); - } - childLbStates.clear(); - } - - private void updateOverallBalancingState() { - ConnectivityState overallState = null; - final Map childPickers = new HashMap<>(); - for (ChildLbState childLbState : childLbStates.values()) { - if (childLbState.deactivated) { - continue; - } - childPickers.put(childLbState.name, childLbState.currentPicker); - overallState = aggregateState(overallState, childLbState.currentState); - } - if (overallState != null) { - SubchannelPicker picker = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - String clusterName = - args.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY); - SubchannelPicker delegate = childPickers.get(clusterName); - if (delegate == null) { - return - PickResult.withError( - Status.UNAVAILABLE.withDescription("CDS encountered error: unable to find " - + "available subchannel for cluster " + clusterName)); - } - return delegate.pickSubchannel(args); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("pickers", childPickers).toString(); - } - }; - helper.updateBalancingState(overallState, picker); - } - } - - @Nullable - private static ConnectivityState aggregateState( - @Nullable ConnectivityState overallState, ConnectivityState childState) { - if (overallState == null) { - return childState; - } - if (overallState == READY || childState == READY) { - return READY; - } - if (overallState == CONNECTING || childState == CONNECTING) { - return CONNECTING; - } - if (overallState == IDLE || childState == IDLE) { - return IDLE; - } - return overallState; - } - - private final class ChildLbState { - private final String name; - private final GracefulSwitchLoadBalancer lb; - private LoadBalancerProvider policyProvider; - private ConnectivityState currentState = CONNECTING; - private SubchannelPicker currentPicker = BUFFER_PICKER; - private boolean deactivated; - @Nullable - ScheduledHandle deletionTimer; - - ChildLbState(String name, LoadBalancerProvider policyProvider) { - this.name = name; - this.policyProvider = policyProvider; - lb = new GracefulSwitchLoadBalancer(new ChildLbStateHelper()); - lb.switchTo(policyProvider); - } - - void deactivate() { - if (deactivated) { - return; - } - - class DeletionTask implements Runnable { - @Override - public void run() { - shutdown(); - childLbStates.remove(name); - } - } - - deletionTimer = - syncContext.schedule( - new DeletionTask(), - DELAYED_CHILD_DELETION_TIME_MINUTES, - TimeUnit.MINUTES, - timeService); - deactivated = true; - logger.log(XdsLogLevel.DEBUG, "Child balancer {0} deactivated", name); - } - - void reactivate(LoadBalancerProvider policyProvider) { - if (deletionTimer != null && deletionTimer.isPending()) { - deletionTimer.cancel(); - deactivated = false; - logger.log(XdsLogLevel.DEBUG, "Child balancer {0} reactivated", name); - } - if (!this.policyProvider.getPolicyName().equals(policyProvider.getPolicyName())) { - logger.log( - XdsLogLevel.DEBUG, - "Child balancer {0} switching policy from {1} to {2}", - name, this.policyProvider.getPolicyName(), policyProvider.getPolicyName()); - lb.switchTo(policyProvider); - this.policyProvider = policyProvider; - } - } - - void shutdown() { - if (deletionTimer != null && deletionTimer.isPending()) { - deletionTimer.cancel(); - } - lb.shutdown(); - logger.log(XdsLogLevel.DEBUG, "Child balancer {0} deleted", name); - } - - private final class ChildLbStateHelper extends ForwardingLoadBalancerHelper { - + protected SubchannelPicker getSubchannelPicker(Map childPickers) { + return new SubchannelPicker() { @Override - public void updateBalancingState(final ConnectivityState newState, - final SubchannelPicker newPicker) { - // If we are already in the process of resolving addresses, the overall balancing state - // will be updated at the end of it, and we don't need to trigger that update here. - if (!childLbStates.containsKey(name)) { - return; - } - // Subchannel picker and state are saved, but will only be propagated to the channel - // when the child instance exits deactivated state. - currentState = newState; - currentPicker = newPicker; - if (!deactivated && !resolvingAddresses) { - updateOverallBalancingState(); + public PickResult pickSubchannel(PickSubchannelArgs args) { + String clusterName = + args.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY); + SubchannelPicker childPicker = childPickers.get(clusterName); + if (childPicker == null) { + return + PickResult.withError( + Status.UNAVAILABLE.withDescription("CDS encountered error: unable to find " + + "available subchannel for cluster " + clusterName)); } + return childPicker.pickSubchannel(args); } @Override - protected Helper delegate() { - return helper; + public String toString() { + return MoreObjects.toStringHelper(this).add("pickers", childPickers).toString(); } - } + }; } } diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java index 3af58ef93cb..a7564e89a8c 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java @@ -55,7 +55,6 @@ import io.grpc.xds.XdsClient.ResourceWatcher; import io.grpc.xds.XdsEndpointResource.EdsUpdate; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; diff --git a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java index e833b3777b8..5cf54317565 100644 --- a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java @@ -21,7 +21,6 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; import io.grpc.ConnectivityState; import io.grpc.InternalLogId; @@ -36,7 +35,6 @@ import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -149,7 +147,7 @@ private void tryNextPriority() { ChildLbState child = new ChildLbState(priority, priorityConfigs.get(priority).ignoreReresolution); children.put(priority, child); - updateOverallState(priority, CONNECTING, BUFFER_PICKER); + updateOverallState(priority, CONNECTING, LoadBalancer.EMPTY_PICKER); // Calling the child's updateResolvedAddresses() can result in tryNextPriority() being // called recursively. We need to be sure to be done with processing here before it is // called. @@ -210,7 +208,7 @@ private final class ChildLbState { @Nullable ScheduledHandle deletionTimer; @Nullable String policy; ConnectivityState connectivityState = CONNECTING; - SubchannelPicker picker = BUFFER_PICKER; + SubchannelPicker picker = LoadBalancer.EMPTY_PICKER; ChildLbState(final String priority, boolean ignoreReresolution) { this.priority = priority; diff --git a/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java b/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java index 436eca8ec5d..20a70cb0322 100644 --- a/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java @@ -39,7 +39,6 @@ import io.grpc.Status; import io.grpc.SynchronizationContext; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.SocketAddress; import java.util.ArrayList; import java.util.Collections; diff --git a/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java b/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java index 825e4a8eca0..596247b8234 100644 --- a/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancer.java @@ -21,7 +21,6 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; import com.google.common.collect.ImmutableMap; import io.grpc.ConnectivityState; @@ -34,7 +33,6 @@ import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -159,7 +157,7 @@ private void updateOverallBalancingState() { if (overallState == TRANSIENT_FAILURE) { picker = new WeightedRandomPicker(errorPickers); } else { - picker = XdsSubchannelPickers.BUFFER_PICKER; + picker = LoadBalancer.EMPTY_PICKER; } } else { picker = new WeightedRandomPicker(childPickers); @@ -191,7 +189,7 @@ private static ConnectivityState aggregateState( private final class ChildHelper extends ForwardingLoadBalancerHelper { String name; ConnectivityState currentState = CONNECTING; - SubchannelPicker currentPicker = BUFFER_PICKER; + SubchannelPicker currentPicker = LoadBalancer.EMPTY_PICKER; private ChildHelper(String name) { this.name = name; diff --git a/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java b/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java index b9196492624..885844f1cbf 100644 --- a/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/WrrLocalityLoadBalancer.java @@ -32,7 +32,6 @@ import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/xds/src/main/java/io/grpc/xds/XdsResourceType.java b/xds/src/main/java/io/grpc/xds/XdsResourceType.java index 86fbf3fd6b9..1bec72d492c 100644 --- a/xds/src/main/java/io/grpc/xds/XdsResourceType.java +++ b/xds/src/main/java/io/grpc/xds/XdsResourceType.java @@ -58,7 +58,7 @@ abstract class XdsResourceType { static boolean enableWrr = getFlag("GRPC_EXPERIMENTAL_XDS_WRR_LB", true); @VisibleForTesting - static boolean enablePickFirst = getFlag("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", false); + static boolean enablePickFirst = getFlag("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", true); static final String TYPE_URL_CLUSTER_CONFIG = "type.googleapis.com/envoy.extensions.clusters.aggregate.v3.ClusterConfig"; diff --git a/xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java b/xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java deleted file mode 100644 index 5c2890c34ea..00000000000 --- a/xds/src/main/java/io/grpc/xds/XdsSubchannelPickers.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.base.MoreObjects; -import io.grpc.LoadBalancer.PickResult; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.Status; - -final class XdsSubchannelPickers { - - private XdsSubchannelPickers() { /* DO NOT CALL ME */ } - - static final SubchannelPicker BUFFER_PICKER = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withNoResult(); - } - - @Override - public String toString() { - return "BUFFER_PICKER"; - } - }; - - static final class ErrorPicker extends SubchannelPicker { - - private final Status error; - - ErrorPicker(Status error) { - this.error = checkNotNull(error, "error"); - } - - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withError(error); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("error", error) - .toString(); - } - } -} diff --git a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java index 6cb12550b60..6eab6151477 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java @@ -54,7 +54,6 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.testing.TestMethodDescriptors; import io.grpc.xds.ClusterManagerLoadBalancerProvider.ClusterManagerConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -126,10 +125,14 @@ public void handleResolvedAddressesUpdatesChannelPicker() { assertThat(pickSubchannel(picker, "childA")).isEqualTo(PickResult.withNoResult()); assertThat(pickSubchannel(picker, "childB")).isEqualTo(PickResult.withNoResult()); assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); - assertThat(childBalancer1.name).isEqualTo("policy_a"); - assertThat(childBalancer2.name).isEqualTo("policy_b"); + assertThat(childBalancers.stream() + .filter(b -> b.name.equals("policy_a")) + .count()).isEqualTo(1); + assertThat(childBalancers.stream() + .filter(b -> b.name.equals("policy_b")) + .count()).isEqualTo(1); + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); assertThat(childBalancer1.config).isEqualTo(lbConfigInventory.get("childA")); assertThat(childBalancer2.config).isEqualTo(lbConfigInventory.get("childB")); @@ -151,8 +154,7 @@ public void handleResolvedAddressesUpdatesChannelPicker() { assertThat(childBalancer2.shutdown).isFalse(); assertThat(childBalancers).hasSize(3); - FakeLoadBalancer childBalancer3 = childBalancers.get(2); - assertThat(childBalancer3.name).isEqualTo("policy_c"); + FakeLoadBalancer childBalancer3 = getChildBalancerByName("policy_c"); assertThat(childBalancer3.config).isEqualTo(lbConfigInventory.get("childC")); // delayed policy_b deletion @@ -166,8 +168,8 @@ public void updateBalancingStateFromChildBalancers() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); Subchannel subchannel1 = mock(Subchannel.class); Subchannel subchannel2 = mock(Subchannel.class); childBalancer1.deliverSubchannelState(subchannel1, ConnectivityState.READY); @@ -184,11 +186,20 @@ public void updateBalancingStateFromChildBalancers() { .isEqualTo(subchannel2); } + private FakeLoadBalancer getChildBalancerByName(String name) { + for (FakeLoadBalancer childLb : childBalancers) { + if (childLb.name.equals(name)) { + return childLb; + } + } + return null; + } + @Test public void ignoreBalancingStateUpdateForDeactivatedChildLbs() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); deliverResolvedAddresses(ImmutableMap.of("childB", "policy_b")); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); // policy_a (deactivated) + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); // policy_a (deactivated) Subchannel subchannel = mock(Subchannel.class); childBalancer1.deliverSubchannelState(subchannel, ConnectivityState.READY); verify(helper, never()).updateBalancingState( @@ -231,8 +242,8 @@ public void handleNameResolutionError_beforeChildLbsInstantiated_returnErrorPick public void handleNameResolutionError_afterChildLbsInstantiated_propagateToChildLbs() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); clusterManagerLoadBalancer.handleNameResolutionError( Status.UNAVAILABLE.withDescription("resolver error")); assertThat(childBalancer1.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); @@ -245,8 +256,8 @@ public void handleNameResolutionError_afterChildLbsInstantiated_propagateToChild public void handleNameResolutionError_notPropagateToDeactivatedChildLbs() { deliverResolvedAddresses(ImmutableMap.of("childA", "policy_a", "childB", "policy_b")); deliverResolvedAddresses(ImmutableMap.of("childB", "policy_b")); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); // policy_a (deactivated) - FakeLoadBalancer childBalancer2 = childBalancers.get(1); // policy_b + FakeLoadBalancer childBalancer1 = getChildBalancerByName("policy_a"); // policy_a (deactivated) + FakeLoadBalancer childBalancer2 = getChildBalancerByName("policy_b"); // policy_b clusterManagerLoadBalancer.handleNameResolutionError( Status.UNKNOWN.withDescription("unknown error")); assertThat(childBalancer1.upstreamError).isNull(); diff --git a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java index a005f40fad7..ebcd68dc950 100644 --- a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java @@ -21,7 +21,7 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; +import static io.grpc.LoadBalancer.EMPTY_PICKER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; @@ -41,6 +41,7 @@ import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.ErrorPicker; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; @@ -55,7 +56,6 @@ import io.grpc.internal.TestUtils.StandardLoadBalancerProvider; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.ArrayList; @@ -423,13 +423,13 @@ public void idleToConnectingDoesNotTriggerFailOver() { // p0 gets IDLE. helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 goes to CONNECTING helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // no failover happened @@ -459,15 +459,15 @@ public void connectingResetFailOverIfSeenReadyOrIdleSinceTransientFailure() { // p0 gets IDLE. helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 goes to CONNECTING, reset failover timer fakeClock.forwardTime(5, TimeUnit.SECONDS); helper0.updateBalancingState( CONNECTING, - BUFFER_PICKER); - verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + EMPTY_PICKER); + verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // failover happens fakeClock.forwardTime(10, TimeUnit.SECONDS); @@ -509,7 +509,7 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { // p0 goes to CONNECTING helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // no failover happened @@ -560,7 +560,7 @@ public void typicalPriorityFailOverFlowWithIdleUpdate() { // p0 gets IDLE. helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 fails over to p1 immediately. @@ -581,13 +581,13 @@ public void typicalPriorityFailOverFlowWithIdleUpdate() { // p2 gets IDLE helper2.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 gets back to IDLE helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p2 fails but does not affect overall picker @@ -614,13 +614,13 @@ public void typicalPriorityFailOverFlowWithIdleUpdate() { // p2 gets back to IDLE helper2.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 gets back to IDLE helper0.updateBalancingState( IDLE, - BUFFER_PICKER); + EMPTY_PICKER); assertCurrentPickerIsBufferPicker(); // p0 fails over to p2 and picker is updated to p2's existing picker. diff --git a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java index 91ab1e8fac4..6cec8b0fb6f 100644 --- a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java @@ -20,7 +20,7 @@ import static io.grpc.ConnectivityState.CONNECTING; import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; +import static io.grpc.LoadBalancer.EMPTY_PICKER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeast; @@ -39,6 +39,7 @@ import io.grpc.Attributes; import io.grpc.EquivalentAddressGroup; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.ErrorPicker; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; @@ -52,7 +53,6 @@ import io.grpc.xds.WeightedRandomPicker.WeightedChildPicker; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.ArrayList; @@ -209,7 +209,7 @@ public void handleResolvedAddresses() { .setAttributes(Attributes.newBuilder().set(fakeKey, fakeValue).build()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); assertThat(childBalancers).hasSize(4); assertThat(childHelpers).hasSize(4); assertThat(fooLbCreated).isEqualTo(2); @@ -246,7 +246,7 @@ public void handleResolvedAddresses() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(newTargets)) .build()); - verify(helper, atLeast(2)).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper, atLeast(2)).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); assertThat(childBalancers).hasSize(5); assertThat(childHelpers).hasSize(5); assertThat(fooLbCreated).isEqualTo(3); // One more foo LB created for target4 @@ -288,7 +288,7 @@ public void handleNameResolutionError() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // Error after child balancers created. weightedTargetLb.handleNameResolutionError(Status.ABORTED); @@ -315,7 +315,7 @@ public void balancingStateUpdatedFromChildBalancers() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // Subchannels to be created for each child balancer. final SubchannelPicker[] subchannelPickers = new SubchannelPicker[]{ @@ -335,7 +335,7 @@ public void balancingStateUpdatedFromChildBalancers() { childHelpers.get(1).updateBalancingState(TRANSIENT_FAILURE, failurePickers[1]); verify(helper, never()).updateBalancingState( eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper, times(2)).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // Another child balancer goes to READY. childHelpers.get(2).updateBalancingState(READY, subchannelPickers[2]); @@ -396,7 +396,7 @@ public void raceBetweenShutdownAndChildLbBalancingStateUpdate() { .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(new WeightedTargetConfig(targets)) .build()); - verify(helper).updateBalancingState(eq(CONNECTING), eq(BUFFER_PICKER)); + verify(helper).updateBalancingState(eq(CONNECTING), eq(EMPTY_PICKER)); // LB shutdown and subchannel state change can happen simultaneously. If shutdown runs first, // any further balancing state update should be ignored. diff --git a/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java index 344876aa348..bb80635f1bd 100644 --- a/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/WrrLocalityLoadBalancerTest.java @@ -30,6 +30,7 @@ import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.ErrorPicker; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.ResolvedAddresses; import io.grpc.LoadBalancer.SubchannelPicker; @@ -41,7 +42,6 @@ import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; import io.grpc.xds.WrrLocalityLoadBalancer.WrrLocalityConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.net.SocketAddress; import java.util.Collections; import java.util.List; diff --git a/xds/third_party/envoy/import.sh b/xds/third_party/envoy/import.sh index c4b5a8516f3..b85b0667800 100755 --- a/xds/third_party/envoy/import.sh +++ b/xds/third_party/envoy/import.sh @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update VERSION then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=main # import VERSION from the google internal copybara_version.txt for Envoy VERSION=0478eba2a495027bf6ac8e787c42e2f5b9eb553b -GIT_REPO="https://github.com/envoyproxy/envoy.git" -GIT_BASE_DIR=envoy -SOURCE_PROTO_BASE_DIR=envoy/api +DOWNLOAD_URL="https://github.com/envoyproxy/envoy/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="envoy-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}/api" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( @@ -181,19 +180,13 @@ envoy/type/v3/semantic_version.proto pushd `git rev-parse --show-toplevel`/xds/third_party/envoy -# clone the envoy github repo in a tmp directory +# put the repo in a tmp directory tmpdir="$(mktemp -d)" trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -pushd "${tmpdir}" -git clone -b $BRANCH $GIT_REPO -trap "rm -rf $GIT_BASE_DIR" EXIT -cd "$GIT_BASE_DIR" -git checkout $VERSION -popd - -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE -cp -p "${tmpdir}/${GIT_BASE_DIR}/NOTICE" NOTICE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/NOTICE" NOTICE rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" diff --git a/xds/third_party/googleapis/import.sh b/xds/third_party/googleapis/import.sh index e83536564e1..d51893e23d4 100755 --- a/xds/third_party/googleapis/import.sh +++ b/xds/third_party/googleapis/import.sh @@ -13,14 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update VERSION then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=master VERSION=ca1372c6d7bcb199638ebfdb40d2b2660bab7b88 -GIT_REPO="https://github.com/googleapis/googleapis.git" -GIT_BASE_DIR=googleapis -SOURCE_PROTO_BASE_DIR=googleapis +DOWNLOAD_URL="https://github.com/googleapis/googleapis/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="googleapis-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( @@ -30,18 +29,12 @@ google/api/expr/v1alpha1/syntax.proto pushd `git rev-parse --show-toplevel`/xds/third_party/googleapis -# clone the googleapis github repo in a tmp directory +# put the repo in a tmp directory tmpdir="$(mktemp -d)" trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -pushd "${tmpdir}" -git clone -b $BRANCH $GIT_REPO -trap "rm -rf $GIT_BASE_DIR" EXIT -cd "$GIT_BASE_DIR" -git checkout $VERSION -popd - -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" diff --git a/xds/third_party/protoc-gen-validate/import.sh b/xds/third_party/protoc-gen-validate/import.sh index 4e30b0e1180..64c92b16c20 100755 --- a/xds/third_party/protoc-gen-validate/import.sh +++ b/xds/third_party/protoc-gen-validate/import.sh @@ -13,33 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update GIT_ORIGIN_REV_ID then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=main -# import GIT_ORIGIN_REV_ID from one of the google internal CLs -GIT_ORIGIN_REV_ID=dfcdc5ea103dda467963fb7079e4df28debcfd28 -GIT_REPO="https://github.com/envoyproxy/protoc-gen-validate.git" -GIT_BASE_DIR=protoc-gen-validate -SOURCE_PROTO_BASE_DIR=protoc-gen-validate +# import VERSION from one of the google internal CLs +VERSION=dfcdc5ea103dda467963fb7079e4df28debcfd28 +DOWNLOAD_URL="https://github.com/envoyproxy/protoc-gen-validate/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="protoc-gen-validate-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( validate/validate.proto ) -# clone the protoc-gen-validate github repo in a tmp directory +pushd `git rev-parse --show-toplevel`/xds/third_party/protoc-gen-validate + +# put the repo in a tmp directory tmpdir="$(mktemp -d)" -pushd "${tmpdir}" -rm -rf "$GIT_BASE_DIR" -git clone -b $BRANCH $GIT_REPO -cd "$GIT_BASE_DIR" -git checkout $GIT_ORIGIN_REV_ID -popd +trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE -cp -p "${tmpdir}/${GIT_BASE_DIR}/NOTICE" NOTICE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/NOTICE" NOTICE +rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}" pushd "${TARGET_PROTO_BASE_DIR}" @@ -51,4 +49,4 @@ do done popd -rm -rf "$tmpdir" +popd diff --git a/xds/third_party/xds/import.sh b/xds/third_party/xds/import.sh index f759cb0d35f..cda86d0368f 100755 --- a/xds/third_party/xds/import.sh +++ b/xds/third_party/xds/import.sh @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update VERSION then in this directory run ./import.sh +# Update VERSION then execute this script set -e -BRANCH=main # import VERSION from one of the google internal CLs VERSION=e9ce68804cb4e64cab5a52e3c8baf840d4ff87b7 -GIT_REPO="https://github.com/cncf/xds.git" -GIT_BASE_DIR=xds -SOURCE_PROTO_BASE_DIR=xds +DOWNLOAD_URL="https://github.com/cncf/xds/archive/${VERSION}.tar.gz" +DOWNLOAD_BASE_DIR="xds-${VERSION}" +SOURCE_PROTO_BASE_DIR="${DOWNLOAD_BASE_DIR}" TARGET_PROTO_BASE_DIR=src/main/proto # Sorted alphabetically. FILES=( @@ -54,18 +53,12 @@ xds/type/v3/typed_struct.proto pushd `git rev-parse --show-toplevel`/xds/third_party/xds -# clone the xds github repo in a tmp directory +# put the repo in a tmp directory tmpdir="$(mktemp -d)" -trap "rm -rf $tmpdir" EXIT +trap "rm -rf ${tmpdir}" EXIT +curl -Ls "${DOWNLOAD_URL}" | tar xz -C "${tmpdir}" -pushd "${tmpdir}" -git clone -b $BRANCH $GIT_REPO -trap "rm -rf $GIT_BASE_DIR" EXIT -cd "$GIT_BASE_DIR" -git checkout $VERSION -popd - -cp -p "${tmpdir}/${GIT_BASE_DIR}/LICENSE" LICENSE +cp -p "${tmpdir}/${DOWNLOAD_BASE_DIR}/LICENSE" LICENSE rm -rf "${TARGET_PROTO_BASE_DIR}" mkdir -p "${TARGET_PROTO_BASE_DIR}"