Skip to content

Commit

Permalink
noMarkingRegionUnavailableOnHttpTimeout (Azure#37163)
Browse files Browse the repository at this point in the history 
* stopMarkingRegionUnavailableForHttpTimeout

---------

Co-authored-by: annie-mac <xinlian@microsoft.com>
  • Loading branch information
@xinlian12
xinlian12 and annie-mac authored Oct 13, 2023
1 parent 96ac934 commit 8b6f7e4
Show file tree
Hide file tree
Showing 4 changed files with 423 additions and 106 deletions.
143 changes: 44 additions & 99 deletions ...ure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ClientRetryPolicyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import io.reactivex.subscribers.TestSubscriber;
import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import reactor.core.publisher.Mono;

Expand All @@ -23,6 +24,21 @@

public class ClientRetryPolicyTest {
private final static int TIMEOUT = 10000;
private final static String TEST_DOCUMENT_PATH = "/dbs/db/colls/col/docs/doc";
private final static String TEST_DATABASE_PATH = "/dbs/db";

@DataProvider(name = "operationProvider")
public static Object[][] operationProvider() {
return new Object[][]{
// OperationType, ResourceType, isAddressRequest, RequestFullPath, ShouldRetryCrossRegion
{ OperationType.Read, ResourceType.Document, Boolean.FALSE, TEST_DOCUMENT_PATH, Boolean.TRUE },
{ OperationType.Read, ResourceType.Document, Boolean.TRUE, TEST_DOCUMENT_PATH, Boolean.FALSE },
{ OperationType.Create, ResourceType.Document, Boolean.FALSE, TEST_DOCUMENT_PATH, Boolean.FALSE },
{ OperationType.Read, ResourceType.Database, Boolean.FALSE, TEST_DATABASE_PATH, Boolean.TRUE },
{ OperationType.Create, ResourceType.Database, Boolean.FALSE, TEST_DATABASE_PATH, Boolean.FALSE },
{ OperationType.QueryPlan, ResourceType.Document, Boolean.FALSE, TEST_DOCUMENT_PATH, Boolean.TRUE }
};
}

@Test(groups = "unit")
public void networkFailureOnRead() throws Exception {
Expand Down Expand Up @@ -58,13 +74,24 @@ public void networkFailureOnRead() throws Exception {
}
}

@Test(groups = "unit")
public void shouldRetryOnGatewayTimeout() throws Exception {
@Test(groups = { "unit" }, dataProvider = "operationProvider")
public void shouldRetryOnGatewayTimeout(
OperationType operationType,
ResourceType resourceType,
boolean isAddressRefresh,
String resourceFullPath,
boolean shouldCrossRegionRetry) throws Exception {
ThrottlingRetryOptions throttlingRetryOptions = new ThrottlingRetryOptions();
GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class);
Mockito.doReturn(new URI("http://localhost")).when(endpointManager).resolveServiceEndpoint(Mockito.any(RxDocumentServiceRequest.class));
Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(true));
ClientRetryPolicy clientRetryPolicy = new ClientRetryPolicy(mockDiagnosticsClientContext(), endpointManager, true, throttlingRetryOptions, null);
ClientRetryPolicy clientRetryPolicy =
new ClientRetryPolicy(
mockDiagnosticsClientContext(),
endpointManager,
true,
throttlingRetryOptions,
null);

Exception exception = ReadTimeoutException.INSTANCE;
CosmosException cosmosException = BridgeInternal.createCosmosException(null, HttpConstants.StatusCodes.REQUEST_TIMEOUT, exception);
Expand All @@ -73,71 +100,23 @@ public void shouldRetryOnGatewayTimeout() throws Exception {
RxDocumentServiceRequest dsr;
Mono<ShouldRetryResult> shouldRetry;

//Data Plane Read
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
OperationType.Read, "/dbs/db/colls/col/docs/doc", ResourceType.Document);

clientRetryPolicy.onBeforeSendRequest(dsr);
shouldRetry = clientRetryPolicy.shouldRetry(cosmosException);

validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(true)
.backOffTime(Duration.ofMillis(1000))
.build());

//Metadata Read
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
OperationType.Read, "/dbs/db/clls/col/docs/doc", ResourceType.Database);

clientRetryPolicy.onBeforeSendRequest(dsr);

shouldRetry = clientRetryPolicy.shouldRetry(cosmosException);

validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(true)
.backOffTime(Duration.ofMillis(1000))
.build());

//Query Plan
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
OperationType.QueryPlan, "/dbs/db/colls/col/docs/doc", ResourceType.Document);

clientRetryPolicy.onBeforeSendRequest(dsr);

shouldRetry = clientRetryPolicy.shouldRetry(cosmosException);

validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(true)
.backOffTime(Duration.ofMillis(1000))
.build());

//Data Plane Write - Should not retry
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
OperationType.Create, "/dbs/db/colls/col/docs/doc", ResourceType.Document);

clientRetryPolicy.onBeforeSendRequest(dsr);
shouldRetry = clientRetryPolicy.shouldRetry(cosmosException);

validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(false)
.build());

//Metadata Write - Should not Retry
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
OperationType.Create, "/dbs/db/colls/col/docs/docId", ResourceType.Database);
dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(), operationType, resourceFullPath, resourceType);
dsr.setAddressRefresh(isAddressRefresh, false);

clientRetryPolicy.onBeforeSendRequest(dsr);

shouldRetry = clientRetryPolicy.shouldRetry(cosmosException);

validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(false)
.build());
if (shouldCrossRegionRetry) {
validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(true)
.backOffTime(Duration.ofMillis(1000))
.build());
} else {
validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(false)
.build());
}
}

@Test(groups = "unit")
Expand Down Expand Up @@ -412,40 +391,6 @@ public void tcpNetworkFailureOnDelete() throws Exception {
}
}

@Test(groups = "unit")
public void httpNetworkFailureOnQueryPlan() throws Exception {
ThrottlingRetryOptions retryOptions = new ThrottlingRetryOptions();
GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class);
Mockito.doReturn(new URI("http://localhost")).when(endpointManager).resolveServiceEndpoint(Mockito.any(RxDocumentServiceRequest.class));
Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(true));
Mockito.doReturn(2).when(endpointManager).getPreferredLocationCount();
ClientRetryPolicy clientRetryPolicy = new ClientRetryPolicy(mockDiagnosticsClientContext(), endpointManager, true, retryOptions, null);

Exception exception = ReadTimeoutException.INSTANCE;
CosmosException cosmosException =
BridgeInternal.createCosmosException(null, HttpConstants.StatusCodes.REQUEST_TIMEOUT, exception);
BridgeInternal.setSubStatusCode(cosmosException, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT);

RxDocumentServiceRequest dsr = RxDocumentServiceRequest.createFromName(mockDiagnosticsClientContext(),
OperationType.QueryPlan, "/dbs/db/colls/col/docs/", ResourceType.Document);
dsr.requestContext = new DocumentServiceRequestContext();

clientRetryPolicy.onBeforeSendRequest(dsr);

for (int i = 0; i < 10; i++) {
Mono<ShouldRetryResult> shouldRetry = clientRetryPolicy.shouldRetry(cosmosException);

validateSuccess(shouldRetry, ShouldRetryValidator.builder()
.nullException()
.shouldRetry(true)
.backOffTime(Duration.ofMillis(1000))
.build());

Mockito.verify(endpointManager, Mockito.times(i+1)).markEndpointUnavailableForRead(Mockito.any());
Mockito.verify(endpointManager, Mockito.times(0)).markEndpointUnavailableForWrite(Mockito.any());
}
}

@Test(groups = "unit")
public void onBeforeSendRequestNotInvoked() {
ThrottlingRetryOptions throttlingRetryOptions = new ThrottlingRetryOptions();
Expand Down
Loading

0 comments on commit 8b6f7e4

Please sign in to comment.