diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index a5c945ff1f..8e43cc17c2 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -918,8 +918,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) { this.ValidateDirectTCPSettings(); this.ValidateLimitToEndpointSettings(); - this.ValidatePartitionLevelFailoverSettings(); - this.ValidateAvailabilityStrategy(); + this.ValidatePartitionLevelFailoverSettings(); ConnectionPolicy connectionPolicy = new ConnectionPolicy() { @@ -1096,15 +1095,6 @@ private void ValidatePartitionLevelFailoverSettings() { throw new ArgumentException($"{nameof(this.ApplicationPreferredRegions)} is required when {nameof(this.EnablePartitionLevelFailover)} is enabled."); } - } - - private void ValidateAvailabilityStrategy() - { - if (this.AvailabilityStrategy != null - && this.ApplicationPreferredRegions == null && this.ApplicationRegion == null) - { - throw new ArgumentException($"{nameof(this.ApplicationPreferredRegions)} or {nameof(this.ApplicationRegion)} must be set to use {nameof(this.AvailabilityStrategy)}"); - } } private void ValidateDirectTCPSettings() diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 3c0a0c7633..4cd873c0c7 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -94,8 +94,16 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con public ReadOnlyCollection AccountReadEndpoints => this.locationCache.AccountReadEndpoints; public ReadOnlyCollection WriteEndpoints => this.locationCache.WriteEndpoints; + + public int PreferredLocationCount + { + get + { + IList effectivePreferredLocations = this.GetEffectivePreferredLocations(); - public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0; + return effectivePreferredLocations.Count; + } + } public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request) { @@ -273,8 +281,7 @@ private async Task TryGetAccountPropertiesFromAllLocationsAsync() return; } - await this.GetAndUpdateAccountPropertiesAsync( - endpoint: serviceEndpoint); + await this.GetAndUpdateAccountPropertiesAsync(endpoint: serviceEndpoint); } } @@ -495,8 +502,8 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account if (this.cancellationTokenSource.IsCancellationRequested) { return; - } - + } + this.locationCache.OnDatabaseAccountRead(databaseAccount); if (this.isBackgroundAccountRefreshActive) @@ -544,7 +551,7 @@ public virtual async Task RefreshLocationAsync(bool forceRefresh = false) public bool CanSupportMultipleWriteLocations(DocumentServiceRequest request) { return this.locationCache.CanUseMultipleWriteLocations() - && this.locationCache.GetAvailableWriteLocations()?.Count > 1 + && this.locationCache.GetAvailableAccountLevelWriteLocations()?.Count > 1 && (request.ResourceType == ResourceType.Document || (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.Execute)); } @@ -647,7 +654,10 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh) try { this.LastBackgroundRefreshUtc = DateTime.UtcNow; - this.locationCache.OnDatabaseAccountRead(await this.GetDatabaseAccountAsync(true)); + AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true); + + this.locationCache.OnDatabaseAccountRead(accountProperties); + } catch (Exception ex) { @@ -671,7 +681,7 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh obsoleteValue: null, singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, - this.connectionPolicy.PreferredLocations, + this.GetEffectivePreferredLocations(), this.connectionPolicy.AccountInitializationCustomEndpoints, this.GetDatabaseAccountAsync, this.cancellationTokenSource.Token), @@ -689,6 +699,17 @@ private bool SkipRefresh(bool forceRefresh) TimeSpan timeSinceLastRefresh = DateTime.UtcNow - this.LastBackgroundRefreshUtc; return (this.isAccountRefreshInProgress || this.MinTimeBetweenAccountRefresh > timeSinceLastRefresh) && !forceRefresh; + } + + public IList GetEffectivePreferredLocations() + { + if (this.connectionPolicy.PreferredLocations != null && this.connectionPolicy.PreferredLocations.Count > 0) + { + return this.connectionPolicy.PreferredLocations; + } + + return this.connectionPolicy.PreferredLocations?.Count > 0 ? + this.connectionPolicy.PreferredLocations : this.locationCache.EffectivePreferredLocations; } } } diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index cc108a722e..79cc336869 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -125,6 +125,8 @@ public ReadOnlyCollection WriteEndpoints } } + public ReadOnlyCollection EffectivePreferredLocations => this.locationInfo.EffectivePreferredLocations; + /// /// Returns the location corresponding to the endpoint if location specific endpoint is provided. /// For the defaultEndPoint, we will return the first available write location. @@ -257,16 +259,22 @@ public Uri GetHubUri() string writeLocation = currentLocationInfo.AvailableWriteLocations[0]; Uri locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation]; return locationEndpointToRoute; - } - - public ReadOnlyCollection GetAvailableReadLocations() - { - return this.locationInfo.AvailableReadLocations; } - - public ReadOnlyCollection GetAvailableWriteLocations() - { - return this.locationInfo.AvailableWriteLocations; + + /// + /// Gets available (account-level) read locations. + /// + public ReadOnlyCollection GetAvailableAccountLevelReadLocations() + { + return this.locationInfo.AvailableReadLocations; + } + + /// + /// Gets available (account-level) write locations. + /// + public ReadOnlyCollection GetAvailableAccountLevelWriteLocations() + { + return this.locationInfo.AvailableWriteLocations; } /// @@ -325,27 +333,37 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request) public ReadOnlyCollection GetApplicableEndpoints(DocumentServiceRequest request, bool isReadRequest) { - ReadOnlyCollection endpoints = - isReadRequest - ? this.ReadEndpoints - : this.WriteEndpoints; - if (request.RequestContext.ExcludeRegions == null || request.RequestContext.ExcludeRegions.Count == 0) { - return endpoints; + return isReadRequest ? this.ReadEndpoints : this.WriteEndpoints; } - return this.GetApplicableEndpoints( + DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; + + ReadOnlyCollection effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; + + return GetApplicableEndpoints( isReadRequest ? this.locationInfo.AvailableReadEndpointByLocation : this.locationInfo.AvailableWriteEndpointByLocation, + effectivePreferredLocations, this.defaultEndpoint, request.RequestContext.ExcludeRegions); - } - + } + public ReadOnlyCollection GetApplicableRegions(IEnumerable excludeRegions, bool isReadRequest) { - return this.GetApplicableRegions( - isReadRequest ? this.locationInfo.AvailableReadLocations : this.locationInfo.AvailableWriteLocations, - this.locationInfo.PreferredLocations[0], + DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; + + ReadOnlyCollection effectivePreferredLocations = this.locationInfo.EffectivePreferredLocations; + + if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) + { + throw new ArgumentException("effectivePreferredLocations cannot be null or empty!"); + } + + return GetApplicableRegions( + isReadRequest ? databaseAccountLocationsInfoSnapshot.AvailableReadLocations : databaseAccountLocationsInfoSnapshot.AvailableWriteLocations, + effectivePreferredLocations, + effectivePreferredLocations[0], excludeRegions); } @@ -353,82 +371,82 @@ public ReadOnlyCollection GetApplicableRegions(IEnumerable exclu /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint /// /// + /// /// /// /// a list of applicable endpoints for a request - private ReadOnlyCollection GetApplicableEndpoints( + private static ReadOnlyCollection GetApplicableEndpoints( ReadOnlyDictionary regionNameByEndpoint, + ReadOnlyCollection effectivePreferredLocations, Uri fallbackEndpoint, IEnumerable excludeRegions) - { - List applicableEndpoints = new List(regionNameByEndpoint.Count); - HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); + { + List applicableEndpoints = new List(regionNameByEndpoint.Count); + HashSet excludeRegionsHash = excludeRegions == null ? new HashSet() : new HashSet(excludeRegions); - if (excludeRegions != null) + foreach (string region in effectivePreferredLocations) { - foreach (string region in this.locationInfo.PreferredLocations) + if (excludeRegionsHash.Count > 0) { - if (!excludeRegionsHash.Contains(region) - && regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) + if (!excludeRegionsHash.Contains(region) && regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) { applicableEndpoints.Add(endpoint); } } - } - else - { - foreach (string region in this.locationInfo.PreferredLocations) + else { - if (regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) - { - applicableEndpoints.Add(endpoint); - } - } - } - - if (applicableEndpoints.Count == 0) - { - applicableEndpoints.Add(fallbackEndpoint); - } - - return new ReadOnlyCollection(applicableEndpoints); - } - + if (regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) + { + applicableEndpoints.Add(endpoint); + } + } + } + + if (applicableEndpoints.Count == 0) + { + applicableEndpoints.Add(fallbackEndpoint); + } + + return new ReadOnlyCollection(applicableEndpoints); + } + /// /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint /// - /// + /// + /// /// /// /// a list of applicable endpoints for a request - private ReadOnlyCollection GetApplicableRegions( - ReadOnlyCollection regionNameByEndpoint, + private static ReadOnlyCollection GetApplicableRegions( + ReadOnlyCollection availableLocations, + ReadOnlyCollection effectivePreferredLocations, string fallbackRegion, IEnumerable excludeRegions) { - List applicableRegions = new List(regionNameByEndpoint.Count); - HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); - - if (excludeRegions != null) - { - foreach (string region in this.locationInfo.PreferredLocations) - { - if (regionNameByEndpoint.Contains(region) - && !excludeRegionsHash.Contains(region)) - { - applicableRegions.Add(region); - } - } - } - else - { - foreach (string region in this.locationInfo.PreferredLocations) - { - if (regionNameByEndpoint.Contains(region)) - { - applicableRegions.Add(region); - } - } + List applicableRegions = new List(availableLocations.Count); + HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); + + if (excludeRegions != null) + { + foreach (string region in effectivePreferredLocations) + { + if (availableLocations.Contains(region) + && !excludeRegionsHash.Contains(region)) + { + applicableRegions.Add(region); + } + } + } + else + { + foreach (string region in effectivePreferredLocations) + { + if (availableLocations.Contains(region)) + { + applicableRegions.Add(region); + } + } } if (applicableRegions.Count == 0) @@ -444,7 +462,7 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) canRefreshInBackground = true; DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); + string mostPreferredLocation = currentLocationInfo.EffectivePreferredLocations.FirstOrDefault(); // we should schedule refresh in background if we are unable to target the user's most preferredLocation. if (this.enableEndpointDiscovery) @@ -648,19 +666,23 @@ private void UpdateLocationCache( { nextLocationInfo.AvailableReadEndpointByLocation = this.GetEndpointByLocation( readLocations, - out ReadOnlyCollection availableReadLocations); + out ReadOnlyCollection availableReadLocations, + out ReadOnlyDictionary availableReadLocationsByEndpoint); nextLocationInfo.AvailableReadLocations = availableReadLocations; nextLocationInfo.AccountReadEndpoints = nextLocationInfo.AvailableReadEndpointByLocation.Select(x => x.Value).ToList().AsReadOnly(); + nextLocationInfo.AvailableReadLocationByEndpoint = availableReadLocationsByEndpoint; } if (writeLocations != null) { nextLocationInfo.AvailableWriteEndpointByLocation = this.GetEndpointByLocation( writeLocations, - out ReadOnlyCollection availableWriteLocations); + out ReadOnlyCollection availableWriteLocations, + out ReadOnlyDictionary availableWriteLocationsByEndpoint); nextLocationInfo.AvailableWriteLocations = availableWriteLocations; + nextLocationInfo.AvailableWriteLocationByEndpoint = availableWriteLocationsByEndpoint; } nextLocationInfo.WriteEndpoints = this.GetPreferredAvailableEndpoints( @@ -673,7 +695,28 @@ private void UpdateLocationCache( endpointsByLocation: nextLocationInfo.AvailableReadEndpointByLocation, orderedLocations: nextLocationInfo.AvailableReadLocations, expectedAvailableOperation: OperationType.Read, - fallbackEndpoint: nextLocationInfo.WriteEndpoints[0]); + fallbackEndpoint: nextLocationInfo.WriteEndpoints[0]); + + nextLocationInfo.EffectivePreferredLocations = nextLocationInfo.PreferredLocations; + + if (nextLocationInfo.PreferredLocations == null || nextLocationInfo.PreferredLocations.Count == 0) + { + if (!nextLocationInfo.AvailableReadLocationByEndpoint.TryGetValue(this.defaultEndpoint, out string regionForDefaultEndpoint)) + { + nextLocationInfo.EffectivePreferredLocations = nextLocationInfo.AvailableReadLocations; + } + else + { + // if defaultEndpoint equals a regional endpoint - do not use account-level regions, + // stick to defaultEndpoint configured for the CosmosClient instance + List locations = new () + { + regionForDefaultEndpoint + }; + + nextLocationInfo.EffectivePreferredLocations = new ReadOnlyCollection(locations); + } + } this.lastCacheUpdateTimestamp = DateTime.UtcNow; @@ -702,17 +745,45 @@ private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionar // If client can use multiple write locations, preferred locations list should be used for determining // both read and write endpoints order. - foreach (string location in currentLocationInfo.PreferredLocations) + if (currentLocationInfo.PreferredLocations != null && currentLocationInfo.PreferredLocations.Count >= 1) { - if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) + foreach (string location in currentLocationInfo.PreferredLocations) { - if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) { - unavailableEndpoints.Add(endpoint); + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } } - else - { - endpoints.Add(endpoint); + } + } + else + { + foreach (string location in orderedLocations) + { + if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) + { + // if defaultEndpoint equals a regional endpoint - do not use account-level regions, + // stick to defaultEndpoint configured for the CosmosClient instance + if (this.defaultEndpoint.Equals(endpoint)) + { + endpoints = new List(); + break; + } + + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } } } } @@ -746,9 +817,11 @@ private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionar return endpoints.AsReadOnly(); } - private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations) + private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations, out ReadOnlyDictionary availableLocationsByEndpoint) { Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary mutableAvailableLocationsByEndpoint = new Dictionary(); + List parsedLocations = new List(); foreach (AccountRegion location in locations) @@ -759,6 +832,9 @@ private ReadOnlyDictionary GetEndpointByLocation(IEnumerable GetEndpointByLocation(IEnumerable(mutableAvailableLocationsByEndpoint); + return new ReadOnlyDictionary(endpointsByLocation); } @@ -801,9 +879,12 @@ public DatabaseAccountLocationsInfo(ReadOnlyCollection preferredLocation this.AvailableReadLocations = new List().AsReadOnly(); this.AvailableWriteEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + this.AvailableWriteLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); + this.AvailableReadLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); this.WriteEndpoints = new List() { defaultEndpoint }.AsReadOnly(); this.AccountReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); + this.EffectivePreferredLocations = new List().AsReadOnly(); } public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) @@ -813,9 +894,12 @@ public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) this.AvailableReadLocations = other.AvailableReadLocations; this.AvailableWriteEndpointByLocation = other.AvailableWriteEndpointByLocation; this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; + this.AvailableReadLocationByEndpoint = other.AvailableReadLocationByEndpoint; + this.AvailableWriteLocationByEndpoint = other.AvailableWriteLocationByEndpoint; this.WriteEndpoints = other.WriteEndpoints; this.AccountReadEndpoints = other.AccountReadEndpoints; this.ReadEndpoints = other.ReadEndpoints; + this.EffectivePreferredLocations = other.EffectivePreferredLocations; } public ReadOnlyCollection PreferredLocations { get; set; } @@ -823,9 +907,13 @@ public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) public ReadOnlyCollection AvailableReadLocations { get; set; } public ReadOnlyDictionary AvailableWriteEndpointByLocation { get; set; } public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } + public ReadOnlyDictionary AvailableWriteLocationByEndpoint { get; set; } + public ReadOnlyDictionary AvailableReadLocationByEndpoint { get; set; } + public ReadOnlyCollection WriteEndpoints { get; set; } public ReadOnlyCollection ReadEndpoints { get; set; } public ReadOnlyCollection AccountReadEndpoints { get; set; } + public ReadOnlyCollection EffectivePreferredLocations { get; set; } } [Flags] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 0bcf55481c..32173e79bd 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -158,8 +158,10 @@ public async Task TestInitAsync() } [TestMethod] + [DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")] + [DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")] [TestCategory("MultiRegion")] - public async Task AvailabilityStrategyNoTriggerTest() + public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmpty) { FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( id: "responseDely", @@ -197,7 +199,7 @@ public async Task AvailabilityStrategyNoTriggerTest() CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(300), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -225,15 +227,28 @@ public async Task AvailabilityStrategyNoTriggerTest() Assert.IsNotNull(hedgeContext); IReadOnlyCollection hedgeContextList; hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.AreEqual(2, hedgeContextList.Count); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + + if (isPreferredLocationsEmpty) + { + Assert.AreEqual(3, hedgeContextList.Count); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.eastUs)); + } + else + { + Assert.AreEqual(2, hedgeContextList.Count); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + } }; } [TestMethod] + [DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")] + [DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")] [TestCategory("MultiRegion")] - public async Task AvailabilityStrategyRequestOptionsTriggerTest() + public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferredLocationsEmpty) { FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( id: "responseDely", @@ -257,7 +272,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest() CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty? new List() : new List() { "Central US", "North Central US" }, Serializer = this.cosmosSystemTextJsonSerializer }; @@ -290,8 +305,10 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest() } [TestMethod] + [DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")] + [DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")] [TestCategory("MultiRegion")] - public async Task AvailabilityStrategyDisableOverideTest() + public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocationsEmpty) { FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( id: "responseDely", @@ -316,7 +333,7 @@ public async Task AvailabilityStrategyDisableOverideTest() CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -350,52 +367,97 @@ public async Task AvailabilityStrategyDisableOverideTest() [DataTestMethod] [TestCategory("MultiRegion")] - [DataRow("Read", "Read", "Gone", DisplayName = "Read | Gone")] - [DataRow("Read", "Read", "RetryWith", DisplayName = "Read | RetryWith")] - [DataRow("Read", "Read", "InternalServerError", DisplayName = "Read | InternalServerError")] - [DataRow("Read", "Read", "ReadSessionNotAvailable", DisplayName = "Read | ReadSessionNotAvailable")] - [DataRow("Read", "Read", "Timeout", DisplayName = "Read | Timeout")] - [DataRow("Read", "Read", "PartitionIsSplitting", DisplayName = "Read | PartitionIsSplitting")] - [DataRow("Read", "Read", "PartitionIsMigrating", DisplayName = "Read | PartitionIsMigrating")] - [DataRow("Read", "Read", "ServiceUnavailable", DisplayName = "Read | ServiceUnavailable")] - [DataRow("Read", "Read", "ResponseDelay", DisplayName = "Read | ResponseDelay")] - [DataRow("SinglePartitionQuery", "Query", "Gone", DisplayName = "SinglePartitionQuery | Gone")] - [DataRow("SinglePartitionQuery", "Query", "RetryWith", DisplayName = "SinglePartitionQuery | RetryWith")] - [DataRow("SinglePartitionQuery", "Query", "InternalServerError", DisplayName = "SinglePartitionQuery | InternalServerError")] - [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable")] - [DataRow("SinglePartitionQuery", "Query", "Timeout", DisplayName = "SinglePartitionQuery | Timeout")] - [DataRow("SinglePartitionQuery", "Query", "PartitionIsSplitting", DisplayName = "SinglePartitionQuery | PartitionIsSplitting")] - [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", DisplayName = "SinglePartitionQuery | PartitionIsMigrating")] - [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", DisplayName = "SinglePartitionQuery | ServiceUnavailable")] - [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", DisplayName = "SinglePartitionQuery | ResponseDelay")] - [DataRow("CrossPartitionQuery", "Query", "Gone", DisplayName = "CrossPartitionQuery | Gone")] - [DataRow("CrossPartitionQuery", "Query", "RetryWith", DisplayName = "CrossPartitionQuery | RetryWith")] - [DataRow("CrossPartitionQuery", "Query", "InternalServerError", DisplayName = "CrossPartitionQuery | InternalServerError")] - [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable")] - [DataRow("CrossPartitionQuery", "Query", "Timeout", DisplayName = "CrossPartitionQuery | Timeout")] - [DataRow("CrossPartitionQuery", "Query", "PartitionIsSplitting", DisplayName = "CrossPartitionQuery | PartitionIsSplitting")] - [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", DisplayName = "CrossPartitionQuery | PartitionIsMigrating")] - [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", DisplayName = "CrossPartitionQuery | ServiceUnavailable")] - [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", DisplayName = "CrossPartitionQuery | ResponseDelay")] - [DataRow("ReadMany", "ReadMany", "Gone", DisplayName = "ReadMany | Gone")] - [DataRow("ReadMany", "ReadMany", "RetryWith", DisplayName = "ReadMany | RetryWith")] - [DataRow("ReadMany", "ReadMany", "InternalServerError", DisplayName = "ReadMany | InternalServerError")] - [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", DisplayName = "ReadMany | ReadSessionNotAvailable")] - [DataRow("ReadMany", "ReadMany", "Timeout", DisplayName = "ReadMany | Timeout")] - [DataRow("ReadMany", "ReadMany", "PartitionIsSplitting", DisplayName = "ReadMany | PartitionIsSplitting")] - [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", DisplayName = "ReadMany | PartitionIsMigrating")] - [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", DisplayName = "ReadMany | ServiceUnavailable")] - [DataRow("ReadMany", "ReadMany", "ResponseDelay", DisplayName = "ReadMany | ResponseDelay")] - [DataRow("ChangeFeed", "ChangeFeed", "Gone", DisplayName = "ChangeFeed | Gone")] - [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", DisplayName = "ChangeFeed | RetryWith")] - [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", DisplayName = "ChangeFeed | InternalServerError")] - [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", DisplayName = "ChangeFeed | ReadSessionNotAvailable")] - [DataRow("ChangeFeed", "ChangeFeed", "Timeout", DisplayName = "ChangeFeed | Timeout")] - [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsSplitting", DisplayName = "ChangeFeed | PartitionIsSplitting")] - [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", DisplayName = "ChangeFeed | PartitionIsMigrating")] - [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", DisplayName = "ChangeFeed | ServiceUnavailable")] - [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", DisplayName = "ChangeFeed | ResponseDelay")] - public async Task AvailabilityStrategyAllFaultsTests(string operation, string conditonName, string resultName) + [DataRow("Read", "Read", "Gone", false, DisplayName = "Read | Gone | With Preferred Regions")] + [DataRow("Read", "Read", "RetryWith", false, DisplayName = "Read | RetryWith | With Preferred Regions")] + [DataRow("Read", "Read", "InternalServerError", false, DisplayName = "Read | InternalServerError | With Preferred Regions")] + [DataRow("Read", "Read", "ReadSessionNotAvailable", false, DisplayName = "Read | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("Read", "Read", "Timeout", false, DisplayName = "Read | Timeout | With Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsSplitting", false, DisplayName = "Read | PartitionIsSplitting | With Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsMigrating", false, DisplayName = "Read | PartitionIsMigrating | With Preferred Regions")] + [DataRow("Read", "Read", "ServiceUnavailable", false, DisplayName = "Read | ServiceUnavailable | With Preferred Regions")] + [DataRow("Read", "Read", "ResponseDelay", false, DisplayName = "Read | ResponseDelay | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Gone", false, DisplayName = "SinglePartitionQuery | Gone | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "RetryWith", false, DisplayName = "SinglePartitionQuery | RetryWith | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "InternalServerError", false, DisplayName = "SinglePartitionQuery | InternalServerError | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", false, DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Timeout", false, DisplayName = "SinglePartitionQuery | Timeout | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsSplitting", false, DisplayName = "SinglePartitionQuery | PartitionIsSplitting | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", false, DisplayName = "SinglePartitionQuery | PartitionIsMigrating | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", false, DisplayName = "SinglePartitionQuery | ServiceUnavailable | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", false, DisplayName = "SinglePartitionQuery | ResponseDelay | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Gone", false, DisplayName = "CrossPartitionQuery | Gone | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "RetryWith", false, DisplayName = "CrossPartitionQuery | RetryWith | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "InternalServerError", false, DisplayName = "CrossPartitionQuery | InternalServerError | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", false, DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Timeout", false, DisplayName = "CrossPartitionQuery | Timeout | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsSplitting", false, DisplayName = "CrossPartitionQuery | PartitionIsSplitting | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", false, DisplayName = "CrossPartitionQuery | PartitionIsMigrating | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", false, DisplayName = "CrossPartitionQuery | ServiceUnavailable | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", false, DisplayName = "CrossPartitionQuery | ResponseDelay | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Gone", false, DisplayName = "ReadMany | Gone | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "RetryWith", false, DisplayName = "ReadMany | RetryWith | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "InternalServerError", false, DisplayName = "ReadMany | InternalServerError | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", false, DisplayName = "ReadMany | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Timeout", false, DisplayName = "ReadMany | Timeout | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsSplitting", false, DisplayName = "ReadMany | PartitionIsSplitting | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", false, DisplayName = "ReadMany | PartitionIsMigrating | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", false, DisplayName = "ReadMany | ServiceUnavailable | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ResponseDelay", false, DisplayName = "ReadMany | ResponseDelay | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Gone", false, DisplayName = "ChangeFeed | Gone | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", false, DisplayName = "ChangeFeed | RetryWith | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", false, DisplayName = "ChangeFeed | InternalServerError | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", false, DisplayName = "ChangeFeed | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Timeout", false, DisplayName = "ChangeFeed | Timeout | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsSplitting", false, DisplayName = "ChangeFeed | PartitionIsSplitting | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", false, DisplayName = "ChangeFeed | PartitionIsMigrating | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", false, DisplayName = "ChangeFeed | ServiceUnavailable | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", false, DisplayName = "ChangeFeed | ResponseDelay | With Preferred Regions")] + [DataRow("Read", "Read", "Gone", true, DisplayName = "Read | Gone | W/O Preferred Regions")] + [DataRow("Read", "Read", "RetryWith", true, DisplayName = "Read | RetryWith | W/O Preferred Regions")] + [DataRow("Read", "Read", "InternalServerError", true, DisplayName = "Read | InternalServerError | W/O Preferred Regions")] + [DataRow("Read", "Read", "ReadSessionNotAvailable", true, DisplayName = "Read | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("Read", "Read", "Timeout", true, DisplayName = "Read | Timeout | W/O Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsSplitting", true, DisplayName = "Read | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsMigrating", true, DisplayName = "Read | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("Read", "Read", "ServiceUnavailable", true, DisplayName = "Read | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("Read", "Read", "ResponseDelay", true, DisplayName = "Read | ResponseDelay | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Gone", true, DisplayName = "SinglePartitionQuery | Gone | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "RetryWith", true, DisplayName = "SinglePartitionQuery | RetryWith | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "InternalServerError", true, DisplayName = "SinglePartitionQuery | InternalServerError | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", true, DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Timeout", true, DisplayName = "SinglePartitionQuery | Timeout | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsSplitting", true, DisplayName = "SinglePartitionQuery | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", true, DisplayName = "SinglePartitionQuery | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", true, DisplayName = "SinglePartitionQuery | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", true, DisplayName = "SinglePartitionQuery | ResponseDelay | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Gone", true, DisplayName = "CrossPartitionQuery | Gone | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "RetryWith", true, DisplayName = "CrossPartitionQuery | RetryWith | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "InternalServerError", true, DisplayName = "CrossPartitionQuery | InternalServerError | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", true, DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Timeout", true, DisplayName = "CrossPartitionQuery | Timeout | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsSplitting", true, DisplayName = "CrossPartitionQuery | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", true, DisplayName = "CrossPartitionQuery | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", true, DisplayName = "CrossPartitionQuery | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", true, DisplayName = "CrossPartitionQuery | ResponseDelay | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Gone", true, DisplayName = "ReadMany | Gone | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "RetryWith", true, DisplayName = "ReadMany | RetryWith | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "InternalServerError", true, DisplayName = "ReadMany | InternalServerError | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", true, DisplayName = "ReadMany | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Timeout", true, DisplayName = "ReadMany | Timeout | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsSplitting", true, DisplayName = "ReadMany | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", true, DisplayName = "ReadMany | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", true, DisplayName = "ReadMany | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ResponseDelay", true, DisplayName = "ReadMany | ResponseDelay | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Gone", true, DisplayName = "ChangeFeed | Gone | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", true, DisplayName = "ChangeFeed | RetryWith | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", true, DisplayName = "ChangeFeed | InternalServerError | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", true, DisplayName = "ChangeFeed | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Timeout", true, DisplayName = "ChangeFeed | Timeout | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsSplitting", true, DisplayName = "ChangeFeed | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", true, DisplayName = "ChangeFeed | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", true, DisplayName = "ChangeFeed | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", true, DisplayName = "ChangeFeed | ResponseDelay | W/O Preferred Regions")] + public async Task AvailabilityStrategyAllFaultsTests(string operation, string conditonName, string resultName, bool isPreferredLocationsEmpty) { FaultInjectionCondition conditon = this.conditions[conditonName]; IFaultInjectionResult result = this.results[resultName]; @@ -415,7 +477,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -437,9 +499,17 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co case "Read": rule.Enable(); + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + + if (isPreferredLocationsEmpty) + { + itemRequestOptions.ExcludeRegions = new List() { "East US" }; + } + ItemResponse ir = await container.ReadItemAsync( "testId", - new PartitionKey("pk")); + new PartitionKey("pk"), + itemRequestOptions); Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; @@ -458,6 +528,11 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co PartitionKey = new PartitionKey("pk"), }; + if (isPreferredLocationsEmpty) + { + requestOptions.ExcludeRegions = new List() { "East US" }; + } + FeedIterator queryIterator = container.GetItemQueryIterator( new QueryDefinition(queryString), requestOptions: requestOptions); @@ -480,8 +555,18 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co case "CrossPartitionQuery": string crossPartitionQueryString = "SELECT * FROM c"; + + QueryRequestOptions queryRequestOptions = new QueryRequestOptions(); + + if (isPreferredLocationsEmpty) + { + queryRequestOptions.ExcludeRegions = new List() { "East US" }; + } + FeedIterator crossPartitionQueryIterator = container.GetItemQueryIterator( - new QueryDefinition(crossPartitionQueryString)); + new QueryDefinition(crossPartitionQueryString), + null, + queryRequestOptions); rule.Enable(); @@ -502,6 +587,13 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co case "ReadMany": rule.Enable(); + ReadManyRequestOptions readManyRequestOptions = new ReadManyRequestOptions(); + + if (isPreferredLocationsEmpty) + { + readManyRequestOptions.ExcludeRegions = new List() { "East US" }; + } + FeedResponse readManyResponse = await container.ReadManyItemsAsync( new List<(string, PartitionKey)>() { @@ -509,7 +601,8 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co ("testId2", new PartitionKey("pk2")), ("testId3", new PartitionKey("pk3")), ("testId4", new PartitionKey("pk4")) - }); + }, + readManyRequestOptions); Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; @@ -562,12 +655,17 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co [DataTestMethod] [TestCategory("MultiRegion")] - [DataRow("Read", "Read", "ReadStep", DisplayName = "Read | ReadStep")] - [DataRow("SinglePartitionQuery", "Query", "QueryStep", DisplayName = "Query | SinglePartitionQueryStep")] - [DataRow("CrossPartitionQuery", "Query", "QueryStep", DisplayName = "Query | CrossPartitionQueryStep")] - [DataRow("ReadMany", "ReadMany", "ReadManyStep", DisplayName = "ReadMany | ReadManyStep")] - [DataRow("ChangeFeed", "ChangeFeed", "ChangeFeedStep", DisplayName = "ChangeFeed | ChangeFeedStep")] - public async Task AvailabilityStrategyStepTests(string operation, string conditonName1, string conditionName2) + [DataRow("Read", "Read", "ReadStep", false, DisplayName = "Read | ReadStep | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "QueryStep", false, DisplayName = "Query | SinglePartitionQueryStep | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "QueryStep", false, DisplayName = "Query | CrossPartitionQueryStep | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadManyStep", false, DisplayName = "ReadMany | ReadManyStep | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ChangeFeedStep", false, DisplayName = "ChangeFeed | ChangeFeedStep | With Preferred Regions")] + [DataRow("Read", "Read", "ReadStep", true, DisplayName = "Read | ReadStep | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "QueryStep", true, DisplayName = "Query | SinglePartitionQueryStep | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "QueryStep", true, DisplayName = "Query | CrossPartitionQueryStep | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadManyStep", true, DisplayName = "ReadMany | ReadManyStep | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ChangeFeedStep", true, DisplayName = "ChangeFeed | ChangeFeedStep | W/O Preferred Regions")] + public async Task AvailabilityStrategyStepTests(string operation, string conditonName1, string conditionName2, bool isPreferredRegionsEmpty) { FaultInjectionCondition conditon1 = this.conditions[conditonName1]; FaultInjectionCondition conditon2 = this.conditions[conditionName2]; @@ -596,7 +694,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US", "East US" }, + ApplicationPreferredRegions = isPreferredRegionsEmpty ? new List() : new List() { "Central US", "North Central US", "East US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ClientRetryPolicyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ClientRetryPolicyTests.cs index 0c7c332854..8df56bdc95 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ClientRetryPolicyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ClientRetryPolicyTests.cs @@ -296,7 +296,7 @@ public async Task ClientRetryPolicy_NoRetry_SingleMaster_Write_PreferredLocation [TestMethod] public async Task ClientRetryPolicy_NoRetry_SingleMaster_Read_NoPreferredLocationsAsync() { - await this.ValidateConnectTimeoutTriggersClientRetryPolicyAsync(isReadRequest: true, useMultipleWriteLocations: false, usesPreferredLocations: false, shouldHaveRetried: false); + await this.ValidateConnectTimeoutTriggersClientRetryPolicyAsync(isReadRequest: true, useMultipleWriteLocations: false, usesPreferredLocations: false, shouldHaveRetried: true); } [TestMethod] @@ -308,13 +308,13 @@ public async Task ClientRetryPolicy_NoRetry_SingleMaster_Write_NoPreferredLocati [TestMethod] public async Task ClientRetryPolicy_NoRetry_MultiMaster_Read_NoPreferredLocationsAsync() { - await this.ValidateConnectTimeoutTriggersClientRetryPolicyAsync(isReadRequest: true, useMultipleWriteLocations: true, usesPreferredLocations: false, false); + await this.ValidateConnectTimeoutTriggersClientRetryPolicyAsync(isReadRequest: true, useMultipleWriteLocations: true, usesPreferredLocations: false, true); } [TestMethod] public async Task ClientRetryPolicy_NoRetry_MultiMaster_Write_NoPreferredLocationsAsync() { - await this.ValidateConnectTimeoutTriggersClientRetryPolicyAsync(isReadRequest: false, useMultipleWriteLocations: true, usesPreferredLocations: false, false); + await this.ValidateConnectTimeoutTriggersClientRetryPolicyAsync(isReadRequest: false, useMultipleWriteLocations: true, usesPreferredLocations: false, true); } private async Task ValidateConnectTimeoutTriggersClientRetryPolicyAsync( diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index e1c0fa0ca9..1e9132149d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -120,8 +120,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "northcentralus" }, accountInitializationCustomEndpoints: null, - getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), - cancellationTokenSource.Token); + getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), + cancellationTokenSource.Token); Assert.Fail("Previous call should have failed"); } @@ -418,6 +418,70 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() } } + /// + /// Test to validate for a client that has been warmed up with account-level regions, any subsequent + /// DatabaseAccount refresh calls should go through the effective preferred regions / account-level read regions + /// if the DatabaseAccount refresh call to the global / default endpoint failed with HttpRequestException (timeout also but not possible to inject + /// w/o adding a refresh method just for this test) + /// + [TestMethod] + public async Task GetDatabaseAccountFromEffectiveRegionalEndpointTestAsync() + { + AccountProperties databaseAccount = new AccountProperties + { + ReadLocationsInternal = new Collection() + { + new AccountRegion + { + Name = "Location1", + Endpoint = "https://testfailover-location1.documents-test.windows-int.net/" + }, + new AccountRegion + { + Name = "Location2", + Endpoint = "https://testfailover-location2.documents-test.windows-int.net/" + }, + new AccountRegion + { + Name = "Location3", + Endpoint = "https://testfailover-location3.documents-test.windows-int.net/" + }, + } + }; + + Uri defaultEndpoint = new Uri("https://testfailover.documents-test.windows-int.net/"); + Uri effectivePreferredRegion1SuffixedUri = new Uri("https://testfailover-location1.documents-test.windows-int.net/"); + + //Setup mock owner "document client" + Mock mockOwner = new Mock(); + + mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(defaultEndpoint); + mockOwner.SetupSequence(owner => + owner.GetDatabaseAccountInternalAsync(defaultEndpoint, It.IsAny())) + .ReturnsAsync(databaseAccount) + .ThrowsAsync(new HttpRequestException()); + mockOwner.Setup(owner => + owner.GetDatabaseAccountInternalAsync(effectivePreferredRegion1SuffixedUri, It.IsAny())) + .ReturnsAsync(databaseAccount); + + // Create connection policy with no preferred locations + ConnectionPolicy connectionPolicy = new ConnectionPolicy(); + + using GlobalEndpointManager globalEndpointManager = + new GlobalEndpointManager(mockOwner.Object, connectionPolicy); + globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount); + + await Task.Delay(TimeSpan.FromSeconds(5)); + await globalEndpointManager.RefreshLocationAsync(forceRefresh: true); + + mockOwner.Verify( + owner => owner.GetDatabaseAccountInternalAsync(defaultEndpoint, It.IsAny()), + Times.Exactly(2)); + mockOwner.Verify( + owner => owner.GetDatabaseAccountInternalAsync(effectivePreferredRegion1SuffixedUri, It.IsAny()), + Times.Once); + } + /// /// Test to validate that when an exception is thrown during a RefreshLocationAsync call /// the exception should not be bubbled up and remain unobserved. The exception should be diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index f5070672d5..f973f2bbec 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -27,6 +27,7 @@ namespace Microsoft.Azure.Cosmos.Client.Tests public sealed class LocationCacheTests { private static Uri DefaultEndpoint = new Uri("https://default.documents.azure.com"); + private static Uri DefaultRegionalEndpoint = new Uri("https://location1.documents.azure.com"); private static Uri Location1Endpoint = new Uri("https://location1.documents.azure.com"); private static Uri Location2Endpoint = new Uri("https://location2.documents.azure.com"); private static Uri Location3Endpoint = new Uri("https://location3.documents.azure.com"); @@ -48,23 +49,34 @@ public sealed class LocationCacheTests private Mock mockedClient; [TestMethod] + [DataRow(true, false, DisplayName = "Validate write endpoint order with preferred locations as empty and multi-write usage disabled and default endpoint is global endpoint.")] + [DataRow(false, false, DisplayName = "Validate write endpoint order with preferred locations as non-empty and multi-write usage disabled and default endpoint is global endpoint.")] + [DataRow(true, true, DisplayName = "Validate write endpoint order with preferred locations as empty and multi-write usage disabled and default endpoint is regional endpoint.")] + [DataRow(false, true, DisplayName = "Validate write endpoint order with preferred locations as non-empty and multi-write usage disabled and default endpoint is regional endpoint.")] [Owner("atulk")] - public void ValidateWriteEndpointOrderWithClientSideDisableMultipleWriteLocation() + public void ValidateWriteEndpointOrderWithClientSideDisableMultipleWriteLocation(bool isPreferredLocationListEmpty, bool isDefaultEndpointARegionalEndpoint) { - using GlobalEndpointManager endpointManager = this.Initialize(false, true, false); + using GlobalEndpointManager endpointManager = this.Initialize( + useMultipleWriteLocations: false, + enableEndpointDiscovery: true, + isPreferredLocationsListEmpty: isPreferredLocationListEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); + Assert.AreEqual(this.cache.WriteEndpoints[0], LocationCacheTests.Location1Endpoint); Assert.AreEqual(this.cache.WriteEndpoints[1], LocationCacheTests.Location2Endpoint); Assert.AreEqual(this.cache.WriteEndpoints[2], LocationCacheTests.Location3Endpoint); } [TestMethod] + [DataRow(true, DisplayName = "Validate get location with preferred locations as non-empty.")] + [DataRow(false, DisplayName = "Validate get location with preferred locations as empty.")] [Owner("atulk")] - public void ValidateGetLocation() + public void ValidateGetLocation(bool isPreferredLocationListEmpty) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: false, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: true); + isPreferredLocationsListEmpty: isPreferredLocationListEmpty); Assert.AreEqual(this.databaseAccount.WriteLocationsInternal.First().Name, this.cache.GetLocation(LocationCacheTests.DefaultEndpoint)); @@ -111,19 +123,19 @@ public void ValidateTryGetLocationForGatewayDiagnostics() [TestMethod] [Owner("atulk")] - public async Task ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryDisabled() + public async Task ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryDisabled() { - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, false, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, false, true); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, true, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, true, true); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, false, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, false, true); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, true, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, true, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, false, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, false, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, true, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, true, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, false, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, false, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, true, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, true, true); } - private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest) + private async Task ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest) { const bool enableEndpointDiscovery = false; @@ -174,27 +186,27 @@ await BackoffRetryUtility.ExecuteAsync( } private ClientRetryPolicy CreateClientRetryPolicy( - bool enableEndpointDiscovery, + bool enableEndpointDiscovery, bool partitionLevelFailoverEnabled, GlobalEndpointManager endpointManager) { return new ClientRetryPolicy( endpointManager, this.partitionKeyRangeLocationCache, - new RetryOptions(), - enableEndpointDiscovery, + new RetryOptions(), + enableEndpointDiscovery, isPertitionLevelFailoverEnabled: partitionLevelFailoverEnabled); } [TestMethod] [Owner("atulk")] - public async Task ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabled() + public async Task ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabled() { - await this.ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(true); - await this.ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(false); + await this.ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(true); + await this.ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(false); } - private async Task ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(bool isPreferredLocationsListEmpty) + private async Task ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(bool isPreferredLocationsListEmpty) { const bool useMultipleWriteLocations = false; bool enableEndpointDiscovery = true; @@ -259,180 +271,485 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] + [DataRow(false, false, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry w/o preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry with preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry w/o preferredLocations with regional default endpoint.")] + [DataRow(true, true, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry with preferredLocations with regional default endpoint.")] [Owner("atulk")] - public async Task ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteLocationsAndEndpointDiscoveryEnabled() + public async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAndEndpointDiscoveryEnabled(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { - await this.ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteLocationsAsync(); - await this.ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync(); + await this.ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); } - private async Task ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteLocationsAsync() + private async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; - ReadOnlyCollection preferredList = new List() { - "location2", - "location1" - }.AsReadOnly(); + ReadOnlyCollection preferredList = isPreferredLocationsEmpty + ? new List().AsReadOnly() + : new List() { "location2", "location1" }.AsReadOnly(); using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: false, - preferedRegionListOverride: preferredList); + preferedRegionListOverride: preferredList, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: false, endpointManager); - using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) + if (!isPreferredLocationsEmpty) { - int retryCount = 0; - - try + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) { - await BackoffRetryUtility.ExecuteAsync( - () => - { - retryPolicy.OnBeforeSendRequest(request); + int retryCount = 0; - if (retryCount == 0) + try + { + await BackoffRetryUtility.ExecuteAsync( + () => { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + retryPolicy.OnBeforeSendRequest(request); - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 1) - { - // Second request must go to the next preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + if (retryCount == 0) + { + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 2) - { - // Third request must go to first preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the next preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to first preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + + throw notFoundException; + }, + retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(3, retryCount); + } + } + } + else + { + if (!isDefaultEndpointARegionalEndpoint) + { + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations are the account-level read locations + Assert.AreEqual(4, effectivePreferredLocations.Count); + + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) + { + int retryCount = 0; + + try + { + await BackoffRetryUtility.ExecuteAsync(() => { - Assert.Fail(); - } + retryPolicy.OnBeforeSendRequest(request); - retryCount++; + if (retryCount == 0) + { + // First request must go to the first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; - StoreResponseNameValueCollection headers = new(); - headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); - DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the second effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to third effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 3) + { + // Third request must go to fourth effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[3]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 4) + { + // Fourth request must go to first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } - throw notFoundException; - }, - retryPolicy); + retryCount++; - Assert.Fail(); + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + throw notFoundException; + }, retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(5, retryCount); + } + } } - catch (NotFoundException) + else { - DefaultTrace.TraceInformation("Received expected notFoundException"); - Assert.AreEqual(3, retryCount); + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations is just the default regional endpoint + Assert.AreEqual(1, effectivePreferredLocations.Count); + + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) + { + int retryCount = 0; + + try + { + await BackoffRetryUtility.ExecuteAsync(() => + { + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + // First request must go to the first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the second effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + throw notFoundException; + }, retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(2, retryCount); + } + } } } } - private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync() + private async Task ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; - ReadOnlyCollection preferredList = new List() { - "location3", - "location2", - "location1" - }.AsReadOnly(); + ReadOnlyCollection preferredList = isPreferredLocationsEmpty + ? new List().AsReadOnly() + : new List() { "location3", "location2", "location1" }.AsReadOnly(); using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: false, - preferedRegionListOverride: preferredList); + preferedRegionListOverride: preferredList, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: false, endpointManager); - using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) + if (!isPreferredLocationsEmpty) { - int retryCount = 0; - - try + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { - await BackoffRetryUtility.ExecuteAsync( - () => - { - retryPolicy.OnBeforeSendRequest(request); + int retryCount = 0; - if (retryCount == 0) - { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 1) - { - // Second request must go to the next preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 2) - { - // Third request must go to the next preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[2]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 3) + try + { + await BackoffRetryUtility.ExecuteAsync( + () => { - // Fourth request must go to first preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the next preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to the next preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[2]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 3) + { + // Fourth request must go to first preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); + + + throw notFoundException; + }, + retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(4, retryCount); + } + } + } + else + { + if (!isDefaultEndpointARegionalEndpoint) + { + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) + { + int retryCount = 0; + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations are the account-level read locations + Assert.AreEqual(4, effectivePreferredLocations.Count); + + // for regions touched for writes - it will be the first 3 effectivePreferredLocations (location1, location2, location3) + // which are the write regions for the account + try + { + await BackoffRetryUtility.ExecuteAsync(() => { - Assert.Fail(); - } + retryPolicy.OnBeforeSendRequest(request); - retryCount++; + if (retryCount == 0) + { + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the next effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to the next effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 3) + { + // Fourth request must go to first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } - StoreResponseNameValueCollection headers = new(); - headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); - DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); + retryCount++; + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); - throw notFoundException; - }, - retryPolicy); + throw notFoundException; + }, retryPolicy); - Assert.Fail(); + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(4, retryCount); + } + } } - catch (NotFoundException) + else { - DefaultTrace.TraceInformation("Received expected notFoundException"); - Assert.AreEqual(4, retryCount); + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) + { + int retryCount = 0; + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations is just the default regional endpoint + Assert.AreEqual(1, effectivePreferredLocations.Count); + + // for regions touched for writes - it will be the first 3 effectivePreferredLocations (location1, location2, location3) + // which are the write regions for the account + try + { + await BackoffRetryUtility.ExecuteAsync(() => + { + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + throw notFoundException; + }, retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(2, retryCount); + } + } } } } [TestMethod] + [DataRow(false, false, DisplayName = "Validate WriteForbidden retries with preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate WriteForbidden retries w/o preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate WriteForbidden retries with preferredLocations with regional default endpoint.")] + [DataRow(true, true, DisplayName = "Validate WriteForbidden retries w/o preferredLocations with regional default endpoint.")] [Owner("atulk")] - public async Task ValidateRetryOnWriteForbiddenExceptionAsync() + public async Task ValidateRetryOnWriteForbiddenExceptionAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: false, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: false); + isPreferredLocationsListEmpty: isPreferredLocationsEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); + if (isPreferredLocationsEmpty) + { + if (!isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + } + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { request.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange() @@ -454,7 +771,9 @@ await BackoffRetryUtility.ExecuteAsync( { this.mockedClient.ResetCalls(); - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; + Uri expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]] : + LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); @@ -468,8 +787,34 @@ await BackoffRetryUtility.ExecuteAsync( { this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny()), Times.Once); - // Next request must go to next preferred endpoint - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + // Next request must go to next available write endpoint + Uri expectedEndpoint; + + if (isPreferredLocationsEmpty) + { + if (isDefaultEndpointARegionalEndpoint) + { + ReadOnlyCollection availableWriteLocations = + this.cache.GetAvailableAccountLevelWriteLocations(); + + Assert.IsNotNull(availableWriteLocations); + Assert.AreEqual(3, availableWriteLocations.Count); + + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(this.cache.EffectivePreferredLocations.Count, 1); + + expectedEndpoint = LocationCacheTests.EndpointByLocation[availableWriteLocations[1]]; + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + } + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return Task.FromResult(true); @@ -486,27 +831,54 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] + [DataRow(false, false, DisplayName = "Validate DatabaseAccountNotFound retries with preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate DatabaseAccountNotFound retries w/o preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate DatabaseAccountNotFound retries with preferredLocations with global default endpoint.")] + [DataRow(true, true, DisplayName = "Validate DatabaseAccountNotFound retries w/o preferredLocations with global default endpoint.")] [Owner("atulk")] - public async Task ValidateRetryOnDatabaseAccountNotFoundAsync() + public async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: false); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: true); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: false); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: true); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); } - private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest) + private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: enableMultipleWriteLocations, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: false); + isPreferredLocationsListEmpty: isPreferredLocationsEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); + if (isPreferredLocationsEmpty) + { + if (enableMultipleWriteLocations) + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.IsTrue(this.cache.EffectivePreferredLocations.Count == 1); + Assert.IsTrue(this.cache.EffectivePreferredLocations[0] == "location1"); + } + } + else + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.IsTrue(this.cache.EffectivePreferredLocations.Count == 1); + Assert.IsTrue(this.cache.EffectivePreferredLocations[0] == "location1"); + } + } + } + endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1; - + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -519,10 +891,14 @@ await BackoffRetryUtility.ExecuteAsync( retryCount++; retryPolicy.OnBeforeSendRequest(request); + // both retries check for flip-flop behavior b/w first two available write regions + // in case of multi-write enabled end to end (client + account) if (retryCount == 1) { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; - + Uri expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]] : + LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); StoreResponseNameValueCollection headers = new(); @@ -533,8 +909,34 @@ await BackoffRetryUtility.ExecuteAsync( } else if (retryCount == 2) { - // Next request must go to next preferred endpoint - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + // Next request must go to next available write endpoint + Uri expectedEndpoint; + + if (isPreferredLocationsEmpty) + { + if (isDefaultEndpointARegionalEndpoint) + { + ReadOnlyCollection availableWriteLocations = + this.cache.GetAvailableAccountLevelWriteLocations(); + + Assert.IsNotNull(availableWriteLocations); + Assert.AreEqual(3, availableWriteLocations.Count); + + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(this.cache.EffectivePreferredLocations.Count, 1); + + expectedEndpoint = LocationCacheTests.EndpointByLocation[availableWriteLocations[1]]; + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + } + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return Task.FromResult(true); @@ -565,35 +967,49 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] - [DataRow(true, true, true)] - [DataRow(true, false, false)] - [DataRow(true, false, true)] - [DataRow(true, true, false)] - [DataRow(false, false, false)] - [DataRow(false, true, true)] - [DataRow(false, true, false)] - [DataRow(false, true, true)] + [DataRow(true, true, true, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, false, false, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, false, true, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, true, false, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, false, false, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, true, true, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, true, false, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, true, true, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, true, true, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(true, false, false, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(true, false, true, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(true, true, false, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, false, false, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, true, true, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, true, false, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, true, true, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] public async Task ValidateAsync( bool useMultipleWriteEndpoints, bool endpointDiscoveryEnabled, - bool isPreferredListEmpty) + bool isPreferredListEmpty, + bool isDefaultEndpointARegionalEndpoint) { await this.ValidateLocationCacheAsync( useMultipleWriteEndpoints, endpointDiscoveryEnabled, - isPreferredListEmpty); + isPreferredListEmpty, + isDefaultEndpointARegionalEndpoint); } [TestMethod] - public async Task ValidateRetryOnHttpExceptionAsync() + [DataRow(false, false, DisplayName = "Validate retry on HTTP exception retries with preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate retry on HTTP exception retries w/o preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate retry on HTTP exception retries with preferredLocations with regional default endpoint.")] + [DataRow(true, true, DisplayName = "Validate retry on HTTP exception retries w/o preferredLocations with regional default endpoint.")] + public async Task ValidateRetryOnHttpExceptionAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: false); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: true); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: false); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: true); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); } - private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLocations, bool isReadRequest) + private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { ReadOnlyCollection preferredList = new List() { "location2", @@ -603,13 +1019,46 @@ private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLoc using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: enableMultipleWriteLocations, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: false, + isPreferredLocationsListEmpty: isPreferredLocationsEmpty, preferedRegionListOverride: preferredList, - enforceSingleMasterSingleWriteLocation: true); + enforceSingleMasterSingleWriteLocation: true, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); + if (isPreferredLocationsEmpty) + { + if (enableMultipleWriteLocations) + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + } + else + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + } + } + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -629,7 +1078,9 @@ await BackoffRetryUtility.ExecuteAsync( || isReadRequest) { // MultiMaster or Single Master Read can use preferred locations for first request - expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]] + : LocationCacheTests.EndpointByLocation[preferredList[0]]; } else { @@ -649,7 +1100,22 @@ await BackoffRetryUtility.ExecuteAsync( || isReadRequest) { // Next request must go to next preferred endpoint - expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + // [or] back to first effective preferred region in case empty preferred regions and regional default endpoint + if (isPreferredLocationsEmpty) + { + if (isDefaultEndpointARegionalEndpoint) + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]]; + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + } } else { @@ -678,28 +1144,45 @@ await BackoffRetryUtility.ExecuteAsync( } [DataTestMethod] - [DataRow(true, false, false, false, false, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(false, false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, true, false, false, false, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(false, true, false, false, false, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, false, true, true, false, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry")] - [DataRow(false, false, true, false, false, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry")] - [DataRow(false, true, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry")] - [DataRow(true, false, false, false, true, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(false, false, false, false, true, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(true, true, false, false, true, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(false, true, false, false, true, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(true, false, true, true, true, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry")] - [DataRow(false, false, true, true, true, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry")] - [DataRow(true, true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry")] - [DataRow(false, true, true, true, true, DisplayName = "Write request - Multi master - with preferred locations - with partition level failover - should retry")] + [DataRow(true, false, false, true, false, false, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, false, false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry - global default endpoint")] + [DataRow(true, true, false, true, false, false, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, true, false, true, false, false, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry - global default endpoint")] + [DataRow(true, false, true, true, false, false, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, false, true, false, false, false, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry - global default endpoint")] + [DataRow(true, true, true, true, false, false, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, true, true, true, false, false, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(true, false, false, true, true, false, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(false, false, false, true, true, false, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, true, false, true, true, false, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(false, true, false, true, true, false, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, false, true, true, true, false, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry - global default endpoint")] + [DataRow(false, false, true, true, true, false, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, true, true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(false, true, true, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, false, false, false, false, true, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, false, false, false, false, true, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, true, false, false, false, true, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, true, false, false, false, true, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, false, true, true, false, true, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry - regional default endpoint")] + [DataRow(false, false, true, false, false, true, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, true, true, true, false, true, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry - regional default endpoint")] + [DataRow(false, true, true, true, false, true, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry - regional default endpoint")] + [DataRow(true, false, false, false, true, true, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, false, false, false, true, true, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, true, false, false, true, true, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, true, false, false, true, true, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, false, true, true, true, true, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, false, true, true, true, true, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry - regional default endpoint")] + [DataRow(true, true, true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry - regional default endpoint")] + [DataRow(false, true, true, true, true, true, DisplayName = "Write request - Multi master - with preferred locations - with partition level failover - should retry - regional default endpoint")] public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, bool shouldHaveRetried, - bool enablePartitionLevelFailover) + bool enablePartitionLevelFailover, + bool isDefaultEndpointARegionalEndpoint) { const bool enableEndpointDiscovery = true; @@ -714,12 +1197,29 @@ public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( isPreferredLocationsListEmpty: !usesPreferredLocations, enablePartitionLevelFailover: enablePartitionLevelFailover, preferedRegionListOverride: preferredList, - enforceSingleMasterSingleWriteLocation: true); + enforceSingleMasterSingleWriteLocation: true, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); - endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); + endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: enablePartitionLevelFailover, endpointManager); + if (!usesPreferredLocations) + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + } + + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -733,13 +1233,67 @@ await BackoffRetryUtility.ExecuteAsync( if (retryCount == 1) { - if (!usesPreferredLocations) + Uri expectedEndpoint; + + if (usesPreferredLocations) { - Assert.Fail("Should not be retrying if preferredlocations is not being used"); + if (useMultipleWriteLocations) + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + } + else + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + } } - - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; - + else + { + if (useMultipleWriteLocations) + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]]; + } + } + } + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount > 1) @@ -771,18 +1325,27 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] - [DataRow(true, true, true, DisplayName = "Read request - Multi master - with preferred locations")] - [DataRow(true, true, false, DisplayName = "Read request - Multi master - no preferred locations")] - [DataRow(true, false, true, DisplayName = "Read request - Single master - with preferred locations")] - [DataRow(true, false, false, DisplayName = "Read request - Single master - no preferred locations")] - [DataRow(false, true, true, DisplayName = "Write request - Multi master - with preferred locations")] - [DataRow(false, true, false, DisplayName = "Write request - Multi master - no preferred locations")] - [DataRow(false, false, true, DisplayName = "Write request - Single master - with preferred locations")] - [DataRow(false, false, false, DisplayName = "Write request - Single master - no preferred locations")] + [DataRow(true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, true, false, false, DisplayName = "Read request - Multi master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, false, true, false, DisplayName = "Read request - Single master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, false, false, false, DisplayName = "Read request - Single master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, true, false, false, DisplayName = "Write request - Multi master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, false, true, false, DisplayName = "Write request - Single master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(true, true, false, true, DisplayName = "Read request - Multi master - no preferred locations - default endpoint is regional endpoint")] + [DataRow(true, false, true, true, DisplayName = "Read request - Single master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(true, false, false, true, DisplayName = "Read request - Single master - no preferred locations - default endpoint is regional endpoint")] + [DataRow(false, true, true, true, DisplayName = "Write request - Multi master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(false, true, false, true, DisplayName = "Write request - Multi master - no preferred locations - default endpoint is regional endpoint")] + [DataRow(false, false, true, true, DisplayName = "Write request - Single master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(false, false, false, true, DisplayName = "Write request - Single master - no preferred locations - default endpoint is regional endpoint")] public void VerifyRegionExcludedTest( bool isReadRequest, bool useMultipleWriteLocations, - bool usesPreferredLocations) + bool usesPreferredLocations, + bool isDefaultEndpointAlsoRegionEndpoint) { bool enableEndpointDiscovery = true; @@ -815,19 +1378,23 @@ public void VerifyRegionExcludedTest( List> excludeRegionCases = isReadRequest ? new List>() { - new List {"default"}, new List {"default", "location1"}, new List {"default", "location2"}, new List {"default", "location4"}, - new List {"default", "location1", "location2"}, new List {"default", "location1", "location4"}, new List {"default", "location2", "location4"}, - new List {"default", "location1", "location2", "location4"}, new List { "location1" }, new List {"location1", "location2"}, - new List {"location1", "location4"}, new List {"location1", "location2", "location4"},new List { "location2" }, - new List {"location2", "location4"},new List { "location4" }, - } : - new List>() + new List { "location1" }, + new List { "location2" }, + new List { "location4" }, + new List { "location1", "location2" }, + new List { "location1", "location4" }, + new List { "location2", "location4" }, + new List { "location1", "location2", "location4" }, + new List { "location1", "location2", "location3", "location4" }, + } : new List>() { - new List {"default" }, new List {"default", "location1"}, new List {"default", "location2"}, new List {"default", "location3"}, - new List {"default", "location1", "location2"}, new List {"default", "location1", "location3"}, new List {"default", "location2", "location3"}, - new List {"default", "location1", "location2", "location3"}, new List { "location1" }, new List {"location1", "location2"}, - new List {"location1", "location3"}, new List {"location1", "location2", "location3"},new List { "location2" }, - new List {"location2", "location3"},new List { "location3" }, + new List { "location1" }, + new List { "location2" }, + new List { "location3" }, + new List { "location1", "location2" }, + new List { "location1", "location3" }, + new List { "location2", "location3" }, + new List { "location1", "location2", "location3" } }; foreach (List excludeRegions in excludeRegionCases) @@ -835,10 +1402,11 @@ public void VerifyRegionExcludedTest( using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, - isPreferredLocationsListEmpty: false, + isPreferredLocationsListEmpty: !usesPreferredLocations, preferedRegionListOverride: preferredList, enforceSingleMasterSingleWriteLocation: true, - isExcludeRegionsTest: true); + isExcludeRegionsTest: true, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointAlsoRegionEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); @@ -846,11 +1414,25 @@ public void VerifyRegionExcludedTest( using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { request.RequestContext.ExcludeRegions = excludeRegions; + ReadOnlyCollection applicableEndpoints; - ReadOnlyCollection applicableEndpoints = this.cache.GetApplicableEndpoints(request, isReadRequest); + if (!isReadRequest && !useMultipleWriteLocations) + { + List applicableEndpointsInner = new List(1); + + Assert.IsNotNull(this.cache.WriteEndpoints); + Assert.IsTrue(this.cache.WriteEndpoints.Count > 0); + + applicableEndpointsInner.Add(this.cache.WriteEndpoints[0]); + applicableEndpoints = applicableEndpointsInner.AsReadOnly(); + } + else + { + applicableEndpoints = this.cache.GetApplicableEndpoints(request, isReadRequest); + } Uri endpoint = endpointManager.ResolveServiceEndpoint(request); - ReadOnlyCollection applicableRegions = this.GetApplicableRegions(isReadRequest, useMultipleWriteLocations, usesPreferredLocations, excludeRegions); + ReadOnlyCollection applicableRegions = this.GetApplicableRegions(isReadRequest, useMultipleWriteLocations, usesPreferredLocations, excludeRegions, isDefaultEndpointAlsoRegionEndpoint); Assert.AreEqual(applicableRegions.Count, applicableEndpoints.Count); for(int i = 0; i < applicableRegions.Count; i++) @@ -864,11 +1446,12 @@ public void VerifyRegionExcludedTest( } - private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, List excludeRegions) + private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, List excludeRegions, bool isDefaultEndpointARegionalEndpoint) { - if(!isReadRequest && !useMultipleWriteLocations) + // exclusion of write region for single-write maps to first available write region + if (!isReadRequest && !useMultipleWriteLocations) { - return new List() {LocationCacheTests.DefaultEndpoint }.AsReadOnly(); + return new List() { LocationCacheTests.Location1Endpoint }.AsReadOnly(); } Dictionary readWriteLocations = usesPreferredLocations ? @@ -888,42 +1471,52 @@ private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool us } : new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint }, } : isReadRequest ? new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint }, {"location1", LocationCacheTests.Location1Endpoint }, {"location2", LocationCacheTests.Location2Endpoint }, + {"location3", LocationCacheTests.Location3Endpoint }, {"location4", LocationCacheTests.Location4Endpoint }, } : useMultipleWriteLocations ? new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint }, {"location1", LocationCacheTests.Location1Endpoint }, {"location2", LocationCacheTests.Location2Endpoint }, {"location3", LocationCacheTests.Location3Endpoint }, } : new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint} }; List applicableRegions = new List(); - foreach (string region in readWriteLocations.Keys) + // exclude regions applies when + // 1. preferred regions are set + // 2. preferred regions aren't set and default endpoint isn't a regional endpoint + if (usesPreferredLocations || (!usesPreferredLocations && !isDefaultEndpointARegionalEndpoint)) { - if(!excludeRegions.Contains(region)) + foreach (string region in readWriteLocations.Keys) { - applicableRegions.Add(readWriteLocations[region]); + if (!excludeRegions.Contains(region)) + { + applicableRegions.Add(readWriteLocations[region]); + } } - } - + } + if (applicableRegions.Count == 0) { - applicableRegions.Add(LocationCacheTests.DefaultEndpoint); + if (isDefaultEndpointARegionalEndpoint) + { + applicableRegions.Add(LocationCacheTests.DefaultRegionalEndpoint); + } + else + { + applicableRegions.Add(LocationCacheTests.DefaultEndpoint); + } } return applicableRegions.AsReadOnly(); @@ -931,19 +1524,9 @@ private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool us private static AccountProperties CreateDatabaseAccount( bool useMultipleWriteLocations, - bool enforceSingleMasterSingleWriteLocation, - bool isExcludeRegionsTest = false) + bool enforceSingleMasterSingleWriteLocation) { - Collection writeLocations = isExcludeRegionsTest ? - - new Collection() - { - { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, - { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, - { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, - } : - new Collection() + Collection writeLocations = new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, @@ -955,11 +1538,7 @@ private static AccountProperties CreateDatabaseAccount( { // Some pre-existing tests depend on the account having multiple write locations even on single master setup // Newer tests can correctly define a single master account (single write region) without breaking existing tests - writeLocations = isExcludeRegionsTest ? - new Collection() - { - { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } } - } : + writeLocations = new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } } @@ -969,18 +1548,12 @@ private static AccountProperties CreateDatabaseAccount( AccountProperties databaseAccount = new AccountProperties() { EnableMultipleWriteLocations = useMultipleWriteLocations, - ReadLocationsInternal = isExcludeRegionsTest ? - new Collection() - { - { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, - { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, - { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, - } : - new Collection() + // ReadLocations should be a superset of WriteLocations + ReadLocationsInternal = new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, }, WriteLocationsInternal = writeLocations @@ -996,12 +1569,12 @@ private GlobalEndpointManager Initialize( bool enforceSingleMasterSingleWriteLocation = false, // Some tests depend on the Initialize to create an account with multiple write locations, even when not multi master ReadOnlyCollection preferedRegionListOverride = null, bool enablePartitionLevelFailover = false, - bool isExcludeRegionsTest = false) + bool isExcludeRegionsTest = false, + bool isDefaultEndpointARegionalEndpoint = false) { this.databaseAccount = LocationCacheTests.CreateDatabaseAccount( useMultipleWriteLocations, - enforceSingleMasterSingleWriteLocation, - isExcludeRegionsTest); + enforceSingleMasterSingleWriteLocation); if (isPreferredLocationsListEmpty) { @@ -1020,7 +1593,7 @@ private GlobalEndpointManager Initialize( this.cache = new LocationCache( this.preferredLocations, - LocationCacheTests.DefaultEndpoint, + isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint, enableEndpointDiscovery, 10, useMultipleWriteLocations); @@ -1028,7 +1601,7 @@ private GlobalEndpointManager Initialize( this.cache.OnDatabaseAccountRead(this.databaseAccount); this.mockedClient = new Mock(); - this.mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(LocationCacheTests.DefaultEndpoint); + this.mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint); this.mockedClient.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny())).ReturnsAsync(this.databaseAccount); ConnectionPolicy connectionPolicy = new ConnectionPolicy() @@ -1043,7 +1616,7 @@ private GlobalEndpointManager Initialize( } GlobalEndpointManager endpointManager = new GlobalEndpointManager(this.mockedClient.Object, connectionPolicy); - + this.partitionKeyRangeLocationCache = enablePartitionLevelFailover ? new GlobalPartitionEndpointManagerCore(endpointManager) : GlobalPartitionEndpointManagerNoOp.Instance; @@ -1054,16 +1627,31 @@ private GlobalEndpointManager Initialize( private async Task ValidateLocationCacheAsync( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, - bool isPreferredListEmpty) + bool isPreferredListEmpty, + bool isDefaultEndpointARegionalEndpoint) { - for (int writeLocationIndex = 0; writeLocationIndex < 3; writeLocationIndex++) + // hardcoded to represent - (location1, location2, location3) as the write regions (with and without preferred regions set) + int maxWriteLocationIndex = 3; + + // hardcoded to represent - (location1, location2, location3, location4) as the account regions and (location1, location2, location3) + // as the read regions (with preferred regions set) + int maxReadLocationIndex = isPreferredListEmpty ? 4 : 3; + + if (isPreferredListEmpty && isDefaultEndpointARegionalEndpoint) + { + maxWriteLocationIndex = 1; + maxReadLocationIndex = 1; + } + + for (int writeLocationIndex = 0; writeLocationIndex < maxWriteLocationIndex; writeLocationIndex++) { - for (int readLocationIndex = 0; readLocationIndex < 2; readLocationIndex++) + for (int readLocationIndex = 0; readLocationIndex < maxReadLocationIndex; readLocationIndex++) { using GlobalEndpointManager endpointManager = this.Initialize( - useMultipleWriteLocations, - endpointDiscoveryEnabled, - isPreferredListEmpty); + useMultipleWriteLocations: useMultipleWriteLocations, + enableEndpointDiscovery: endpointDiscoveryEnabled, + isPreferredLocationsListEmpty: isPreferredListEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); ReadOnlyCollection currentWriteEndpoints = this.cache.WriteEndpoints; ReadOnlyCollection currentReadEndpoints = this.cache.ReadEndpoints; @@ -1085,23 +1673,55 @@ private async Task ValidateLocationCacheAsync( location => location.Name, location => new Uri(location.Endpoint)); - Dictionary readEndpointByLocation = this.databaseAccount.ReadableRegions.ToDictionary( + Dictionary readEndpointByLocation = this.databaseAccount.ReadLocationsInternal.ToDictionary( location => location.Name, location => new Uri(location.Endpoint)); - Uri[] preferredAvailableWriteEndpoints = this.preferredLocations.Skip(writeLocationIndex) - .Where(location => writeEndpointByLocation.ContainsKey(location)) - .Select(location => writeEndpointByLocation[location]).ToArray(); + List accountLevelReadEndpoints = this.databaseAccount.ReadLocationsInternal + .Where(accountRegion => readEndpointByLocation.ContainsKey(accountRegion.Name)) + .Select(accountRegion => readEndpointByLocation[accountRegion.Name]) + .ToList(); + + List accountLevelWriteEndpoints = this.databaseAccount.WriteLocationsInternal + .Where(accountRegion => writeEndpointByLocation.ContainsKey(accountRegion.Name)) + .Select(accountRegion => writeEndpointByLocation[accountRegion.Name]) + .ToList(); + + ReadOnlyCollection preferredLocationsWhenClientLevelPreferredLocationsIsEmpty = this.cache.EffectivePreferredLocations; + + Uri[] preferredAvailableWriteEndpoints, preferredAvailableReadEndpoints; + + if (isPreferredListEmpty) + { + preferredAvailableWriteEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(writeLocationIndex) + .Where(location => writeEndpointByLocation.ContainsKey(location)) + .Select(location => writeEndpointByLocation[location]).ToArray(); + + preferredAvailableReadEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(readLocationIndex) + .Where(location => readEndpointByLocation.ContainsKey(location)) + .Select(location => readEndpointByLocation[location]).ToArray(); + } + else + { + preferredAvailableWriteEndpoints = this.preferredLocations.Skip(writeLocationIndex) + .Where(location => writeEndpointByLocation.ContainsKey(location)) + .Select(location => writeEndpointByLocation[location]).ToArray(); - Uri[] preferredAvailableReadEndpoints = this.preferredLocations.Skip(readLocationIndex) - .Where(location => readEndpointByLocation.ContainsKey(location)) - .Select(location => readEndpointByLocation[location]).ToArray(); + preferredAvailableReadEndpoints = this.preferredLocations.Skip(readLocationIndex) + .Where(location => readEndpointByLocation.ContainsKey(location)) + .Select(location => readEndpointByLocation[location]).ToArray(); + } this.ValidateEndpointRefresh( useMultipleWriteLocations, endpointDiscoveryEnabled, + isPreferredListEmpty, preferredAvailableWriteEndpoints, preferredAvailableReadEndpoints, + preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, + preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, + accountLevelWriteEndpoints, + accountLevelReadEndpoints, writeLocationIndex > 0, readLocationIndex > 0 && currentReadEndpoints[0] != LocationCacheTests.DefaultEndpoint, @@ -1114,7 +1734,9 @@ private async Task ValidateLocationCacheAsync( useMultipleWriteLocations, endpointDiscoveryEnabled, preferredAvailableWriteEndpoints, - preferredAvailableReadEndpoints); + preferredAvailableReadEndpoints, + isPreferredListEmpty, + isDefaultEndpointARegionalEndpoint); // wait for TTL on unavailability info string expirationTime = System.Configuration.ConfigurationManager.AppSettings["UnavailableLocationsExpirationTimeInSeconds"]; @@ -1151,8 +1773,13 @@ private async Task ValidateLocationCacheAsync( private void ValidateEndpointRefresh( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, + bool isPreferredListEmpty, Uri[] preferredAvailableWriteEndpoints, Uri[] preferredAvailableReadEndpoints, + ReadOnlyCollection preferredAvailableWriteRegions, + ReadOnlyCollection preferredAvailableReadRegions, + List accountLevelWriteEndpoints, + List accountLevelReadEndpoints, bool isFirstWriteEndpointUnavailable, bool isFirstReadEndpointUnavailable, bool hasMoreThanOneWriteEndpoints, @@ -1163,19 +1790,30 @@ private void ValidateEndpointRefresh( bool isMostPreferredLocationUnavailableForRead = isFirstReadEndpointUnavailable; bool isMostPreferredLocationUnavailableForWrite = useMultipleWriteLocations ? false : isFirstWriteEndpointUnavailable; - if (this.preferredLocations.Count > 0) + + if (this.preferredLocations.Count > 0 || (isPreferredListEmpty && endpointDiscoveryEnabled)) { - string mostPreferredReadLocationName = this.preferredLocations.First(location => databaseAccount.ReadableRegions.Any(readLocation => readLocation.Name == location)); + string mostPreferredReadLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableReadRegions[0] : this.preferredLocations.FirstOrDefault(location => this.databaseAccount.ReadableRegions.Any(readLocation => readLocation.Name == location), ""); Uri mostPreferredReadEndpoint = LocationCacheTests.EndpointByLocation[mostPreferredReadLocationName]; isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints.Length == 0 ? true : (preferredAvailableReadEndpoints[0] != mostPreferredReadEndpoint); - string mostPreferredWriteLocationName = this.preferredLocations.First(location => databaseAccount.WritableRegions.Any(writeLocation => writeLocation.Name == location)); + if (isPreferredListEmpty && endpointDiscoveryEnabled) + { + isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints[0] != accountLevelReadEndpoints[0]; + } + + string mostPreferredWriteLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableWriteRegions[0] : this.preferredLocations.FirstOrDefault(location => this.databaseAccount.WritableRegions.Any(writeLocation => writeLocation.Name == location), ""); Uri mostPreferredWriteEndpoint = LocationCacheTests.EndpointByLocation[mostPreferredWriteLocationName]; if (useMultipleWriteLocations) { isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints.Length == 0 ? true : (preferredAvailableWriteEndpoints[0] != mostPreferredWriteEndpoint); } + + if (isPreferredListEmpty && endpointDiscoveryEnabled) + { + isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints[0] != accountLevelWriteEndpoints[0]; + } } if (!endpointDiscoveryEnabled) @@ -1222,15 +1860,17 @@ private void ValidateRequestEndpointResolution( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, Uri[] availableWriteEndpoints, - Uri[] availableReadEndpoints) + Uri[] availableReadEndpoints, + bool isPreferredLocationsListEmpty, + bool isDefaultEndpointARegionalEndpoint) { Uri firstAvailableWriteEndpoint; Uri secondAvailableWriteEndpoint; if (!endpointDiscoveryEnabled) { - firstAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; - secondAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; + firstAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; + secondAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; } else if (!useMultipleWriteLocations) { @@ -1239,32 +1879,45 @@ private void ValidateRequestEndpointResolution( } else if (availableWriteEndpoints.Length > 1) { - firstAvailableWriteEndpoint = availableWriteEndpoints[0]; - secondAvailableWriteEndpoint = availableWriteEndpoints[1]; + + if (isDefaultEndpointARegionalEndpoint && isPreferredLocationsListEmpty) + { + firstAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + secondAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + } + else + { + firstAvailableWriteEndpoint = availableWriteEndpoints[0]; + secondAvailableWriteEndpoint = availableWriteEndpoints[1]; + } } else if (availableWriteEndpoints.Length > 0) { - firstAvailableWriteEndpoint = availableWriteEndpoints[0]; - secondAvailableWriteEndpoint = - this.databaseAccount.WriteLocationsInternal[0].Endpoint != firstAvailableWriteEndpoint.ToString() ? - new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint) : - new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); + if (isDefaultEndpointARegionalEndpoint && isPreferredLocationsListEmpty) + { + firstAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + secondAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + } + else + { + firstAvailableWriteEndpoint = availableWriteEndpoints[0]; + secondAvailableWriteEndpoint = + this.databaseAccount.WriteLocationsInternal[0].Endpoint != firstAvailableWriteEndpoint.ToString() ? + new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint) : + new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); + } } else { - firstAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; - secondAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; + firstAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; + secondAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; } Uri firstAvailableReadEndpoint; if (!endpointDiscoveryEnabled) { - firstAvailableReadEndpoint = LocationCacheTests.DefaultEndpoint; - } - else if (this.preferredLocations.Count == 0) - { - firstAvailableReadEndpoint = firstAvailableWriteEndpoint; + firstAvailableReadEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; } else if (availableReadEndpoints.Length > 0) { @@ -1275,14 +1928,20 @@ private void ValidateRequestEndpointResolution( firstAvailableReadEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; } - Uri firstWriteEnpoint = !endpointDiscoveryEnabled ? + Uri firstWriteEndpoint = !endpointDiscoveryEnabled ? LocationCacheTests.DefaultEndpoint : new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint); - Uri secondWriteEnpoint = !endpointDiscoveryEnabled ? + Uri secondWriteEndpoint = !endpointDiscoveryEnabled ? LocationCacheTests.DefaultEndpoint : new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); + if (isDefaultEndpointARegionalEndpoint && !endpointDiscoveryEnabled) + { + firstWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + secondWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + } + // If current write endpoint is unavailable, write endpoints order doesn't change // All write requests flip-flop between current write and alternate write endpoint ReadOnlyCollection writeEndpoints = this.cache.WriteEndpoints; @@ -1291,8 +1950,8 @@ private void ValidateRequestEndpointResolution( Assert.AreEqual(firstAvailableWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Document, false)); // Writes to other resource types should be directed to first/second write endpoint - Assert.AreEqual(firstWriteEnpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, false)); - Assert.AreEqual(secondWriteEnpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, true)); + Assert.AreEqual(firstWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, false)); + Assert.AreEqual(secondWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, true)); // Reads should be directed to available read endpoints regardless of resource type Assert.AreEqual(firstAvailableReadEndpoint, this.ResolveEndpointForReadRequest(true));